diff --git a/airbyte-ci/connectors/base_images/base_images/commands.py b/airbyte-ci/connectors/base_images/base_images/commands.py index 169420353c70..626724cf61fb 100644 --- a/airbyte-ci/connectors/base_images/base_images/commands.py +++ b/airbyte-ci/connectors/base_images/base_images/commands.py @@ -10,9 +10,10 @@ import dagger import inquirer # type: ignore import semver -from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry from jinja2 import Environment, FileSystemLoader +from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry + async def _generate_docs(dagger_client: dagger.Client): """This function will generate the README.md file from the templates/README.md.j2 template. diff --git a/airbyte-ci/connectors/base_images/base_images/consts.py b/airbyte-ci/connectors/base_images/base_images/consts.py index ce3701c300f1..cdee55c11a7e 100644 --- a/airbyte-ci/connectors/base_images/base_images/consts.py +++ b/airbyte-ci/connectors/base_images/base_images/consts.py @@ -2,10 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -"""This module declares constants used by the base_images module. -""" +"""This module declares constants used by the base_images module.""" import dagger + REMOTE_REGISTRY = "docker.io" PLATFORMS_WE_PUBLISH_FOR = (dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64")) diff --git a/airbyte-ci/connectors/base_images/base_images/errors.py b/airbyte-ci/connectors/base_images/base_images/errors.py index 3e009806c80b..78596195cdb5 100644 --- a/airbyte-ci/connectors/base_images/base_images/errors.py +++ b/airbyte-ci/connectors/base_images/base_images/errors.py @@ -2,8 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -"""This module contains the exceptions used by the base_images module. -""" +"""This module contains the exceptions used by the base_images module.""" from typing import Union diff --git a/airbyte-ci/connectors/base_images/base_images/hacks.py b/airbyte-ci/connectors/base_images/base_images/hacks.py index 1d8ab1a9ad48..5f3bf76d2a1f 100644 --- a/airbyte-ci/connectors/base_images/base_images/hacks.py +++ b/airbyte-ci/connectors/base_images/base_images/hacks.py @@ -4,6 +4,7 @@ import dagger + # If we perform addition dagger operations on the container, we need to make sure that a mapping exists for the new field name. DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION = { "from": lambda field: f'FROM {field.args.get("address")}', diff --git a/airbyte-ci/connectors/base_images/base_images/java/bases.py b/airbyte-ci/connectors/base_images/base_images/java/bases.py index ed820e5b9863..72a376e3aae8 100644 --- a/airbyte-ci/connectors/base_images/base_images/java/bases.py +++ b/airbyte-ci/connectors/base_images/base_images/java/bases.py @@ -6,6 +6,7 @@ from typing import Callable, Final import dagger + from base_images import bases, published_image from base_images import sanity_checks as base_sanity_checks from base_images.python import sanity_checks as python_sanity_checks @@ -22,9 +23,9 @@ class AirbyteJavaConnectorBaseImage(bases.AirbyteConnectorBaseImage): DD_AGENT_JAR_URL: Final[str] = "https://dtdg.co/latest-java-tracer" BASE_SCRIPT_URL = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh" - JAVA_BASE_SCRIPT_URL: Final[ - str - ] = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh" + JAVA_BASE_SCRIPT_URL: Final[str] = ( + "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh" + ) def get_container(self, platform: dagger.Platform) -> dagger.Container: """Returns the container used to build the base image for java connectors diff --git a/airbyte-ci/connectors/base_images/base_images/publish.py b/airbyte-ci/connectors/base_images/base_images/publish.py index 4f4ebcfaf3e9..e6e69fa8b7fe 100644 --- a/airbyte-ci/connectors/base_images/base_images/publish.py +++ b/airbyte-ci/connectors/base_images/base_images/publish.py @@ -4,6 +4,7 @@ import dagger + from base_images import bases, consts, published_image diff --git a/airbyte-ci/connectors/base_images/base_images/python/bases.py b/airbyte-ci/connectors/base_images/base_images/python/bases.py index cedb8eaad3dd..191c05c20574 100644 --- a/airbyte-ci/connectors/base_images/base_images/python/bases.py +++ b/airbyte-ci/connectors/base_images/base_images/python/bases.py @@ -6,6 +6,7 @@ from typing import Callable, Final import dagger + from base_images import bases, published_image from base_images import sanity_checks as base_sanity_checks from base_images.python import sanity_checks as python_sanity_checks diff --git a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py index 58724281a1e2..798839009669 100644 --- a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py @@ -3,6 +3,7 @@ # import dagger + from base_images import errors from base_images import sanity_checks as base_sanity_checks diff --git a/airbyte-ci/connectors/base_images/base_images/root_images.py b/airbyte-ci/connectors/base_images/base_images/root_images.py index dcd0892a8f6c..0134cd4d34e8 100644 --- a/airbyte-ci/connectors/base_images/base_images/root_images.py +++ b/airbyte-ci/connectors/base_images/base_images/root_images.py @@ -4,6 +4,7 @@ from .published_image import PublishedImage + PYTHON_3_9_18 = PublishedImage( registry="docker.io", repository="python", diff --git a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py index 287636cef73c..3ea3b4e9b310 100644 --- a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py @@ -6,6 +6,7 @@ from typing import Optional import dagger + from base_images import errors diff --git a/airbyte-ci/connectors/base_images/base_images/utils/docker.py b/airbyte-ci/connectors/base_images/base_images/utils/docker.py index b8bc51449170..b7618fcb4aed 100644 --- a/airbyte-ci/connectors/base_images/base_images/utils/docker.py +++ b/airbyte-ci/connectors/base_images/base_images/utils/docker.py @@ -9,6 +9,7 @@ from typing import List, Tuple import dagger + from base_images import console, published_image @@ -31,7 +32,6 @@ def get_credentials() -> Tuple[str, str]: class CraneClient: - CRANE_IMAGE_ADDRESS = "gcr.io/go-containerregistry/crane/debug:c195f151efe3369874c72662cd69ad43ee485128@sha256:94f61956845714bea3b788445454ae4827f49a90dcd9dac28255c4cccb6220ad" def __init__(self, dagger_client: dagger.Client, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0): diff --git a/airbyte-ci/connectors/base_images/base_images/version_registry.py b/airbyte-ci/connectors/base_images/base_images/version_registry.py index 1495c77c327c..df2a137a6c17 100644 --- a/airbyte-ci/connectors/base_images/base_images/version_registry.py +++ b/airbyte-ci/connectors/base_images/base_images/version_registry.py @@ -11,6 +11,7 @@ import dagger import semver + from base_images import consts, published_image from base_images.bases import AirbyteConnectorBaseImage from base_images.java.bases import AirbyteJavaConnectorBaseImage @@ -18,6 +19,7 @@ from base_images.utils import docker from connector_ops.utils import ConnectorLanguage # type: ignore + MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage, AirbyteJavaConnectorBaseImage] diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py index 9f2f47456848..6f31626adf7b 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py @@ -11,6 +11,7 @@ from . import SecretsManager + logger = Logger() ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS" diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py index 067e89da1d68..84295b70de45 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py @@ -8,6 +8,7 @@ from dataclasses import dataclass + DEFAULT_SECRET_FILE = "config" diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py index c024caf01587..554d0d3f604f 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py @@ -17,6 +17,7 @@ from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret + DEFAULT_SECRET_FILE_WITH_EXT = DEFAULT_SECRET_FILE + ".json" GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) diff --git a/airbyte-ci/connectors/common_utils/common_utils/google_api.py b/airbyte-ci/connectors/common_utils/common_utils/google_api.py index 68ff38ae5a9f..69d12894ac90 100644 --- a/airbyte-ci/connectors/common_utils/common_utils/google_api.py +++ b/airbyte-ci/connectors/common_utils/common_utils/google_api.py @@ -11,6 +11,7 @@ from .logger import Logger + TOKEN_TTL = 3600 diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py index f109c4299c86..b28bbbf0d1a8 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py @@ -5,8 +5,10 @@ from typing import Dict, List, Optional, Set, Tuple, Union import yaml + from connector_ops import utils + # The breaking change reviewers is still in active use. BREAKING_CHANGE_REVIEWERS = {"breaking-change-reviewers"} CERTIFIED_MANIFEST_ONLY_CONNECTOR_REVIEWERS = {"dev-python"} diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 7eb850c4b880..32e03ebdbcf0 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -22,6 +22,7 @@ from rich.console import Console from simpleeval import simple_eval + console = Console() DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py index ed400f6ef598..1587db395bd9 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py @@ -12,6 +12,7 @@ import dagger from anyio import Semaphore from connector_ops.utils import Connector # type: ignore + from connectors_insights.insights import generate_insights_for_connector from connectors_insights.result_backends import GCSBucket, LocalDir from connectors_insights.utils import gcs_uri_to_bucket_key, get_all_connectors_in_directory, remove_strict_encrypt_suffix diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py index 8cdf36d6b154..5267c88df8ce 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING import requests + from connectors_insights.hacks import get_ci_on_master_report from connectors_insights.models import ConnectorInsights from connectors_insights.pylint import get_pylint_output diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py index d8fc5a4105ab..eef3fe1cd086 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import ConnectorLanguage # type: ignore + from connectors_insights.utils import never_fail_exec if TYPE_CHECKING: diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py index fa0f86795e47..c78308c980d9 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import Connector # type: ignore + from connectors_qa.models import Check, CheckCategory, CheckResult if TYPE_CHECKING: @@ -27,7 +28,6 @@ class CheckConnectorIconIsAvailable(AssetsCheck): requires_metadata = False def _check_is_valid_svg(self, icon_path: Path) -> Tuple[bool, str | None]: - try: # Ensure the file has an .svg extension if not icon_path.suffix.lower() == ".svg": diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py index 700b7162ed27..5665882b52ce 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py @@ -7,9 +7,10 @@ import requests # type: ignore from connector_ops.utils import Connector, ConnectorLanguage # type: ignore -from connectors_qa.models import Check, CheckCategory, CheckResult from pydash.objects import get # type: ignore +from connectors_qa.models import Check, CheckCategory, CheckResult + from .helpers import ( generate_description, prepare_changelog_to_compare, diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py index da2b73b25c72..d7f0cd0c5df7 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py @@ -5,9 +5,10 @@ import toml from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore class MetadataCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py index 806fcb232c2d..608e9b9dc9fb 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py @@ -3,9 +3,10 @@ import semver import toml from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from pydash.objects import get # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from pydash.objects import get # type: ignore class PackagingCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py index 61744ef0009a..623368022530 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py @@ -4,9 +4,10 @@ from typing import Iterable, Optional, Set, Tuple from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from pydash.objects import get # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from pydash.objects import get # type: ignore class SecurityCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py index fb70e431fd91..39860d002a74 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py @@ -2,9 +2,10 @@ from connector_ops.utils import Connector # type: ignore -from connectors_qa.models import Check, CheckCategory, CheckResult from pydash.collections import find # type: ignore +from connectors_qa.models import Check, CheckCategory, CheckResult + class TestingCheck(Check): category = CheckCategory.TESTING diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py index 77ec4a0879d0..1f680c3c12ea 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py @@ -6,11 +6,12 @@ import asyncclick as click import asyncer from connector_ops.utils import Connector # type: ignore +from jinja2 import Environment, PackageLoader, select_autoescape + from connectors_qa.checks import ENABLED_CHECKS from connectors_qa.consts import CONNECTORS_QA_DOC_TEMPLATE_NAME from connectors_qa.models import Check, CheckCategory, CheckResult, CheckStatus, Report from connectors_qa.utils import get_all_connectors_in_directory, remove_strict_encrypt_suffix -from jinja2 import Environment, PackageLoader, select_autoescape # HELPERS diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py index 5d6eb29fe615..5b823a4830d4 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py @@ -11,6 +11,7 @@ from typing import Dict, List, Optional from connector_ops.utils import Connector, ConnectorLanguage # type: ignore + from connectors_qa import consts ALL_LANGUAGES = [ @@ -289,9 +290,9 @@ def to_json(self) -> str: " ", "_" ) connectors_report[connector_technical_name]["badge_text"] = badge_text - connectors_report[connector_technical_name][ - "badge_url" - ] = f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}" + connectors_report[connector_technical_name]["badge_url"] = ( + f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}" + ) return json.dumps( { "generation_timestamp": datetime.utcnow().isoformat(), diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py index a2ee14c8436a..1833e1e05ac0 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py @@ -4,6 +4,7 @@ from typing import Set from connector_ops.utils import Connector # type: ignore + from connectors_qa import consts diff --git a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py index afe74946cb10..f28ad4fd8c5a 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py @@ -6,23 +6,28 @@ import git import pytest from asyncclick.testing import CliRunner + from connectors_qa.cli import generate_documentation DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO = Path("docs/contributing-to-airbyte/resources/qa-checks.md") + @pytest.fixture def airbyte_repo(): return git.Repo(search_parent_directories=True) + @pytest.mark.asyncio async def test_generated_qa_checks_documentation_is_up_to_date(airbyte_repo, tmp_path): # Arrange current_doc = (airbyte_repo.working_dir / DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO).read_text() newly_generated_doc_path = tmp_path / "qa-checks.md" - + # Act await CliRunner().invoke(generate_documentation, [str(tmp_path / "qa-checks.md")], catch_exceptions=False) # Assert suggested_command = f"connectors-qa generate-documentation {DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO}" - assert newly_generated_doc_path.read_text() == current_doc, f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." + assert ( + newly_generated_doc_path.read_text() == current_doc + ), f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py index ed4e8eebbfe9..f26d49671fdd 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py @@ -42,7 +42,6 @@ def test_fail_when_icon_path_is_not_named_icon_svg(self, tmp_path, mocker): assert result.status == CheckStatus.FAILED assert result.message == "Icon file is not a SVG file" - def test_fail_when_icon_file_is_not_valid_svg(self, tmp_path, mocker): # Arrange connector = mocker.MagicMock() diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py index 959ba6b48311..f5f58912489a 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py @@ -196,7 +196,6 @@ def test_pass_when_documentation_file_path_exists(self, mocker, tmp_path): class TestCheckDocumentationContent: - def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_path): # Arrange connector = mocker.Mock( @@ -204,7 +203,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat ab_internal_sl=300, language="python", connector_type="source", - documentation_file_path=tmp_path / "not_existing_documentation.md" + documentation_file_path=tmp_path / "not_existing_documentation.md", ) # Act @@ -217,11 +216,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat def test_fail_when_documentation_file_path_is_none(self, mocker): # Arrange connector = mocker.Mock( - technical_name="test-connector", - ab_internal_sl=300, - language="python", - connector_type="source", - documentation_file_path=None + technical_name="test-connector", ab_internal_sl=300, language="python", connector_type="source", documentation_file_path=None ) # Act @@ -263,34 +258,28 @@ def test_fail_when_documentation_file_has_missing_headers(self, connector_with_i assert "Actual Heading: 'For Airbyte Cloud:'. Expected Heading: 'Setup guide'" in result.message def test_fail_when_documentation_file_not_have_all_required_fields_in_prerequisites_section_content( - self, - connector_with_invalid_documentation + self, connector_with_invalid_documentation ): # Act - result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run( - connector_with_invalid_documentation - ) + result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Missing descriptions for required spec fields: github repositories" in result.message - def test_fail_when_documentation_file_has_invalid_source_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_source_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector GitHub section content does not follow standard template:" in result.message - assert "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." in result.message + assert ( + "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." + in result.message + ) - def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_invalid_documentation) @@ -299,10 +288,7 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_cont assert "Connector For Airbyte Cloud: section content does not follow standard template:" in result.message assert "+ 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account." in result.message - def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_invalid_documentation) @@ -311,23 +297,19 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_conte assert "Connector For Airbyte Open Source: section content does not follow standard template" in result.message assert "+ 1. Navigate to the Airbyte Open Source dashboard." in result.message - def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector Supported sync modes section content does not follow standard template:" in result.message - assert ("+ The GitHub source connector supports the following" - " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):") in result.message + assert ( + "+ The GitHub source connector supports the following" + " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):" + ) in result.message - def test_fail_when_documentation_file_has_invalid_tutorials_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_tutorials_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_invalid_documentation) @@ -336,10 +318,7 @@ def test_fail_when_documentation_file_has_invalid_tutorials_section_content( assert "Connector Tutorials section content does not follow standard template:" in result.message assert "+ Now that you have set up the GitHub source connector, check out the following GitHub tutorials:" in result.message - def test_fail_when_documentation_file_has_invalid_changelog_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_changelog_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_invalid_documentation) @@ -356,10 +335,7 @@ def test_pass_when_documentation_file_has_correct_headers(self, connector_with_c assert result.status == CheckStatus.PASSED assert result.message == "Documentation guidelines are followed" - def test_pass_when_documentation_file_has_correct_prerequisites_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_prerequisites_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_correct_documentation) @@ -367,10 +343,7 @@ def test_pass_when_documentation_file_has_correct_prerequisites_section_content( assert result.status == CheckStatus.PASSED assert "All required fields from spec are present in the connector documentation" in result.message - def test_pass_when_documentation_file_has_correct_source_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_source_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_correct_documentation) @@ -378,10 +351,7 @@ def test_pass_when_documentation_file_has_correct_source_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_correct_documentation) @@ -389,10 +359,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_cont assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_correct_documentation) @@ -400,10 +367,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_conte assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_correct_documentation) @@ -411,10 +375,7 @@ def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_c assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_tutorials_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_tutorials_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_correct_documentation) @@ -422,10 +383,7 @@ def test_pass_when_documentation_file_has_correct_tutorials_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_headers_order( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_headers_order(self, connector_with_correct_documentation): # Act result = documentation.CheckDocumentationHeadersOrder()._run(connector_with_correct_documentation) @@ -433,10 +391,7 @@ def test_pass_when_documentation_file_has_correct_headers_order( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_changelog_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_changelog_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_correct_documentation) diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py index b3b6e952fa44..482a2536b241 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py @@ -4,6 +4,7 @@ import os import pytest + from connectors_qa import consts from connectors_qa.checks import metadata from connectors_qa.models import CheckStatus diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py index b494c9d40074..56f98ae27081 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py @@ -133,6 +133,7 @@ def test_pass_if_publish_to_pypi_is_disabled(self, mocker): assert result.status == CheckStatus.PASSED assert "PyPi publishing is declared" in result.message + class TestCheckConnectorLicense: def test_fail_when_license_is_missing(self, mocker): # Arrange diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py index c6c997b03fd6..99a90b2cb230 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py @@ -2,6 +2,7 @@ import pytest from connector_ops.utils import ConnectorLanguage + from connectors_qa.checks import testing from connectors_qa.models import CheckStatus @@ -56,9 +57,7 @@ "connectorTestSuitesOptions": [ { "suite": testing.AcceptanceTestsEnabledCheck.test_suite_name, - "testSecrets": { - "testSecret": "test" - }, + "testSecrets": {"testSecret": "test"}, }, { "suite": "unit", @@ -71,10 +70,10 @@ OTHER_USAGE_VALUES = ["low", "none", "unknown", None, ""] DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES = [ - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, ] DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES = [ @@ -89,21 +88,9 @@ class TestAcceptanceTestsEnabledCheck: @pytest.mark.parametrize( "cases_to_test, usage_values_to_test, expected_result", [ - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - OTHER_USAGE_VALUES, - CheckStatus.SKIPPED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.PASSED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.FAILED - ) + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, OTHER_USAGE_VALUES, CheckStatus.SKIPPED), + (DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.PASSED), + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.FAILED), ], ) def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases_to_test, usage_values_to_test, expected_result): @@ -115,11 +102,13 @@ def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases metadata=metadata_case, language=ConnectorLanguage.PYTHON, connector_type="source", - ab_internal_sl=100 + ab_internal_sl=100, ) # Act result = testing.AcceptanceTestsEnabledCheck().run(connector) # Assert - assert result.status == expected_result, f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" + assert ( + result.status == expected_result + ), f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py index 442a038f9595..bcb3198e7bca 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from connector_ops.utils import ConnectorLanguage + from connectors_qa import consts from connectors_qa.checks import ENABLED_CHECKS from connectors_qa.models import CheckStatus diff --git a/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py b/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py index 1e80a737c6a2..116e20e1dfe0 100644 --- a/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py +++ b/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py @@ -4,11 +4,22 @@ from typing import List, Set, Union import yaml -from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver -from airbyte_protocol.models import AirbyteCatalog, AirbyteStream # type: ignore # missing library stubs or py.typed marker -from erd.relationships import Relationships +from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ( + ManifestReferenceResolver, +) +from airbyte_protocol.models import ( # type: ignore # missing library stubs or py.typed marker + AirbyteCatalog, + AirbyteStream, +) from pydbml import Database # type: ignore # missing library stubs or py.typed marker -from pydbml.classes import Column, Index, Reference, Table # type: ignore # missing library stubs or py.typed marker +from pydbml.classes import ( # type: ignore # missing library stubs or py.typed marker + Column, + Index, + Reference, + Table, +) + +from erd.relationships import Relationships class Source: @@ -25,33 +36,67 @@ def is_dynamic(self, stream_name: str) -> bool: manifest_static_streams = set() if self._has_manifest(): with open(self._get_manifest_path()) as manifest_file: - resolved_manifest = ManifestReferenceResolver().preprocess_manifest(yaml.safe_load(manifest_file)) + resolved_manifest = ManifestReferenceResolver().preprocess_manifest( + yaml.safe_load(manifest_file) + ) for stream in resolved_manifest["streams"]: if "schema_loader" not in stream: # stream is assumed to have `DefaultSchemaLoader` which will show in the schemas folder so we can skip continue if stream["schema_loader"]["type"] == "InlineSchemaLoader": - name = stream["name"] if "name" in stream else stream.get("$parameters").get("name", None) + name = ( + stream["name"] + if "name" in stream + else stream.get("$parameters").get("name", None) + ) if not name: print(f"Could not retrieve name for this stream: {stream}") continue - manifest_static_streams.add(stream["name"] if "name" in stream else stream.get("$parameters").get("name", None)) + manifest_static_streams.add( + stream["name"] + if "name" in stream + else stream.get("$parameters").get("name", None) + ) - return stream_name not in manifest_static_streams | self._get_streams_from_schemas_folder() + return ( + stream_name + not in manifest_static_streams | self._get_streams_from_schemas_folder() + ) def _get_streams_from_schemas_folder(self) -> Set[str]: - schemas_folder = self._source_folder / self._source_technical_name.replace("-", "_") / "schemas" - return {p.name.replace(".json", "") for p in schemas_folder.iterdir() if p.is_file()} if schemas_folder.exists() else set() + schemas_folder = ( + self._source_folder + / self._source_technical_name.replace("-", "_") + / "schemas" + ) + return ( + { + p.name.replace(".json", "") + for p in schemas_folder.iterdir() + if p.is_file() + } + if schemas_folder.exists() + else set() + ) def _get_manifest_path(self) -> Path: - return self._source_folder / self._source_technical_name.replace("-", "_") / "manifest.yaml" + return ( + self._source_folder + / self._source_technical_name.replace("-", "_") + / "manifest.yaml" + ) def _has_manifest(self) -> bool: return self._get_manifest_path().exists() class DbmlAssembler: - def assemble(self, source: Source, discovered_catalog: AirbyteCatalog, relationships: Relationships) -> Database: + def assemble( + self, + source: Source, + discovered_catalog: AirbyteCatalog, + relationships: Relationships, + ) -> Database: database = Database() for stream in discovered_catalog.streams: if source.is_dynamic(stream.name): @@ -66,7 +111,9 @@ def assemble(self, source: Source, discovered_catalog: AirbyteCatalog, relations def _create_table(self, stream: AirbyteStream) -> Table: dbml_table = Table(stream.name) - for property_name, property_information in stream.json_schema.get("properties").items(): + for property_name, property_information in stream.json_schema.get( + "properties" + ).items(): try: dbml_table.add_column( Column( @@ -79,12 +126,20 @@ def _create_table(self, stream: AirbyteStream) -> Table: print(f"Ignoring field {property_name}: {exception}") continue - if stream.source_defined_primary_key and len(stream.source_defined_primary_key) > 1: + if ( + stream.source_defined_primary_key + and len(stream.source_defined_primary_key) > 1 + ): if any(map(lambda key: len(key) != 1, stream.source_defined_primary_key)): - raise ValueError(f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`") + raise ValueError( + f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`" + ) composite_key_columns = [ - column for key in stream.source_defined_primary_key for column in dbml_table.columns if column.name in key + column + for key in stream.source_defined_primary_key + for column in dbml_table.columns + if column.name in key ] if len(composite_key_columns) < len(stream.source_defined_primary_key): raise ValueError("Unexpected error: missing PK column from dbml table") @@ -97,11 +152,15 @@ def _create_table(self, stream: AirbyteStream) -> Table: ) return dbml_table - def _add_references(self, source: Source, database: Database, relationships: Relationships) -> None: + def _add_references( + self, source: Source, database: Database, relationships: Relationships + ) -> None: for stream in relationships["streams"]: for column_name, relationship in stream["relations"].items(): if source.is_dynamic(stream["name"]): - print(f"Skipping relationship as stream {stream['name']} from relationship is dynamic") + print( + f"Skipping relationship as stream {stream['name']} from relationship is dynamic" + ) continue try: @@ -109,18 +168,26 @@ def _add_references(self, source: Source, database: Database, relationships: Rel ".", 1 ) # we support the field names having dots but not stream name hence we split on the first dot only except ValueError as exception: - raise ValueError(f"Could not handle relationship {relationship}") from exception + raise ValueError( + f"Could not handle relationship {relationship}" + ) from exception if source.is_dynamic(target_table_name): - print(f"Skipping relationship as target stream {target_table_name} is dynamic") + print( + f"Skipping relationship as target stream {target_table_name} is dynamic" + ) continue try: database.add_reference( Reference( type="<>", # we don't have the information of which relationship type it is so we assume many-to-many for now - col1=self._get_column(database, stream["name"], column_name), - col2=self._get_column(database, target_table_name, target_column_name), + col1=self._get_column( + database, stream["name"], column_name + ), + col2=self._get_column( + database, target_table_name, target_column_name + ), ) ) except ValueError as exception: @@ -136,24 +203,38 @@ def _extract_type(self, property_type: Union[str, List[str]]) -> str: # show this in DBML types.remove("null") if len(types) != 1: - raise ValueError(f"Expected only one type apart from `null` but got {len(types)}: {property_type}") + raise ValueError( + f"Expected only one type apart from `null` but got {len(types)}: {property_type}" + ) return types[0] def _is_pk(self, stream: AirbyteStream, property_name: str) -> bool: return stream.source_defined_primary_key == [[property_name]] - def _get_column(self, database: Database, table_name: str, column_name: str) -> Column: - matching_tables = list(filter(lambda dbml_table: dbml_table.name == table_name, database.tables)) + def _get_column( + self, database: Database, table_name: str, column_name: str + ) -> Column: + matching_tables = list( + filter(lambda dbml_table: dbml_table.name == table_name, database.tables) + ) if len(matching_tables) == 0: raise ValueError(f"Could not find table {table_name}") elif len(matching_tables) > 1: - raise ValueError(f"Unexpected error: many tables found with name {table_name}") + raise ValueError( + f"Unexpected error: many tables found with name {table_name}" + ) table: Table = matching_tables[0] - matching_columns = list(filter(lambda column: column.name == column_name, table.columns)) + matching_columns = list( + filter(lambda column: column.name == column_name, table.columns) + ) if len(matching_columns) == 0: - raise ValueError(f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}") + raise ValueError( + f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}" + ) elif len(matching_columns) > 1: - raise ValueError(f"Unexpected error: many columns found with name {column_name} for table {table_name}") + raise ValueError( + f"Unexpected error: many columns found with name {column_name} for table {table_name}" + ) return matching_columns[0] diff --git a/airbyte-ci/connectors/erd/src/erd/erd_service.py b/airbyte-ci/connectors/erd/src/erd/erd_service.py index aff60bd79a9e..47591d365aae 100644 --- a/airbyte-ci/connectors/erd/src/erd/erd_service.py +++ b/airbyte-ci/connectors/erd/src/erd/erd_service.py @@ -7,11 +7,16 @@ import dpath import google.generativeai as genai # type: ignore # missing library stubs or py.typed marker -from airbyte_protocol.models import AirbyteCatalog # type: ignore # missing library stubs or py.typed marker +from airbyte_protocol.models import ( + AirbyteCatalog, # type: ignore # missing library stubs or py.typed marker +) +from markdown_it import MarkdownIt +from pydbml.renderer.dbml.default import ( + DefaultDBMLRenderer, # type: ignore # missing library stubs or py.typed marker +) + from erd.dbml_assembler import DbmlAssembler, Source from erd.relationships import Relationships, RelationshipsMerger -from markdown_it import MarkdownIt -from pydbml.renderer.dbml.default import DefaultDBMLRenderer # type: ignore # missing library stubs or py.typed marker class ErdService: @@ -21,12 +26,18 @@ def __init__(self, source_technical_name: str, source_path: Path) -> None: self._model = genai.GenerativeModel("gemini-1.5-flash") if not self._discovered_catalog_path.exists(): - raise ValueError(f"Could not find discovered catalog at path {self._discovered_catalog_path}") + raise ValueError( + f"Could not find discovered catalog at path {self._discovered_catalog_path}" + ) def generate_estimated_relationships(self) -> None: normalized_catalog = self._normalize_schema_catalog(self._get_catalog()) - estimated_relationships = self._get_relations_from_gemini(source_name=self._source_path.name, catalog=normalized_catalog) - with open(self._estimated_relationships_file, "w") as estimated_relationship_file: + estimated_relationships = self._get_relations_from_gemini( + source_name=self._source_path.name, catalog=normalized_catalog + ) + with open( + self._estimated_relationships_file, "w" + ) as estimated_relationship_file: json.dump(estimated_relationships, estimated_relationship_file, indent=4) def write_dbml_file(self) -> None: @@ -34,7 +45,8 @@ def write_dbml_file(self) -> None: Source(self._source_path, self._source_technical_name), self._get_catalog(), RelationshipsMerger().merge( - self._get_relationships(self._estimated_relationships_file), self._get_relationships(self._confirmed_relationships_file) + self._get_relationships(self._estimated_relationships_file), + self._get_relationships(self._confirmed_relationships_file), ), ) @@ -53,13 +65,19 @@ def _normalize_schema_catalog(catalog: AirbyteCatalog) -> dict[str, Any]: to_rem = dpath.search( stream["json_schema"]["properties"], ["**"], - afilter=lambda x: isinstance(x, dict) and ("array" in str(x.get("type", "")) or "object" in str(x.get("type", ""))), + afilter=lambda x: isinstance(x, dict) + and ( + "array" in str(x.get("type", "")) + or "object" in str(x.get("type", "")) + ), ) for key in to_rem: stream["json_schema"]["properties"].pop(key) return streams # type: ignore # as this comes from an AirbyteCatalog dump, the format should be fine - def _get_relations_from_gemini(self, source_name: str, catalog: dict[str, Any]) -> Relationships: + def _get_relations_from_gemini( + self, source_name: str, catalog: dict[str, Any] + ) -> Relationships: """ :param source_name: @@ -74,9 +92,7 @@ def _get_relations_from_gemini(self, source_name: str, catalog: dict[str, Any]) The current JSON Schema format is as follows: {current_schema}, where "streams" has a list of streams, which represents database tables, and list of properties in each, which in turn, represent DB columns. Streams presented in list are the only available ones. Generate and add a `foreign_key` with reference for each field in top level of properties that is helpful in understanding what the data represents and how are streams related to each other. Pay attention to fields ends with '_id'. - """.format( - source_name=source_name, current_schema=catalog - ) + """.format(source_name=source_name, current_schema=catalog) task = """ Please provide answer in the following format: {streams: [{"name": "", "relations": {"": ""} }]} diff --git a/airbyte-ci/connectors/erd/src/erd/relationships.py b/airbyte-ci/connectors/erd/src/erd/relationships.py index 704af25d489e..432b19981124 100644 --- a/airbyte-ci/connectors/erd/src/erd/relationships.py +++ b/airbyte-ci/connectors/erd/src/erd/relationships.py @@ -16,16 +16,28 @@ class Relationship(TypedDict): class RelationshipsMerger: - def merge(self, estimated_relationships: Relationships, confirmed_relationships: Relationships) -> Relationships: + def merge( + self, + estimated_relationships: Relationships, + confirmed_relationships: Relationships, + ) -> Relationships: streams = [] for estimated_stream in estimated_relationships["streams"]: - confirmed_relationships_for_stream = self._get_stream(confirmed_relationships, estimated_stream["name"]) + confirmed_relationships_for_stream = self._get_stream( + confirmed_relationships, estimated_stream["name"] + ) if confirmed_relationships_for_stream: - streams.append(self._merge_for_stream(estimated_stream, confirmed_relationships_for_stream)) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None + streams.append( + self._merge_for_stream( + estimated_stream, confirmed_relationships_for_stream + ) + ) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None else: streams.append(estimated_stream) - already_processed_streams = set(map(lambda relationship: relationship["name"], streams)) + already_processed_streams = set( + map(lambda relationship: relationship["name"], streams) + ) for confirmed_stream in confirmed_relationships["streams"]: if confirmed_stream["name"] not in already_processed_streams: streams.append( @@ -36,13 +48,20 @@ def merge(self, estimated_relationships: Relationships, confirmed_relationships: ) return {"streams": streams} - def _merge_for_stream(self, estimated: Relationship, confirmed: Relationship) -> Relationship: + def _merge_for_stream( + self, estimated: Relationship, confirmed: Relationship + ) -> Relationship: relations = copy.deepcopy(confirmed.get("relations", {})) # get estimated but filter out false positives for field, target in estimated.get("relations", {}).items(): - false_positives = confirmed["false_positives"] if "false_positives" in confirmed else {} - if field not in relations and (field not in false_positives or false_positives.get(field, None) != target): # type: ignore # at this point, false_positives should not be None + false_positives = ( + confirmed["false_positives"] if "false_positives" in confirmed else {} + ) + if field not in relations and ( + field not in false_positives + or false_positives.get(field, None) != target + ): # type: ignore # at this point, false_positives should not be None relations[field] = target return { @@ -50,7 +69,9 @@ def _merge_for_stream(self, estimated: Relationship, confirmed: Relationship) -> "relations": relations, } - def _get_stream(self, relationships: Relationships, stream_name: str) -> Optional[Relationship]: + def _get_stream( + self, relationships: Relationships, stream_name: str + ) -> Optional[Relationship]: for stream in relationships["streams"]: if stream.get("name", None) == stream_name: return stream diff --git a/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py b/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py index c697505378cd..93c65a702535 100644 --- a/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py +++ b/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py @@ -4,6 +4,7 @@ from unittest.mock import Mock from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, SyncMode + from erd.dbml_assembler import DbmlAssembler, Source from tests.builder import RelationshipBuilder @@ -18,7 +19,9 @@ def setUp(self) -> None: def test_given_no_streams_then_database_is_empty(self) -> None: dbml = self._assembler.assemble( - self._source, AirbyteCatalog(streams=[]), {"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]} + self._source, + AirbyteCatalog(streams=[]), + {"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]}, ) assert not dbml.tables diff --git a/airbyte-ci/connectors/erd/tests/test_relationships.py b/airbyte-ci/connectors/erd/tests/test_relationships.py index f8675cb6bd57..968928eb1763 100644 --- a/airbyte-ci/connectors/erd/tests/test_relationships.py +++ b/airbyte-ci/connectors/erd/tests/test_relationships.py @@ -17,32 +17,72 @@ def setUp(self) -> None: self._merger = RelationshipsMerger() def test_given_no_confirmed_then_return_estimation(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } confirmed: Relationships = {"streams": []} merged = self._merger.merge(estimated, confirmed) assert merged == estimated - def test_given_confirmed_as_false_positive_then_remove_from_estimation(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_false_positive(_A_COLUMN, _A_TARGET).build()]} + def test_given_confirmed_as_false_positive_then_remove_from_estimation( + self, + ) -> None: + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_false_positive(_A_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) assert merged == {"streams": [{"name": "a_stream_name", "relations": {}}]} - def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives(self) -> None: + def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives( + self, + ) -> None: estimated: Relationships = {"streams": []} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) assert merged == confirmed def test_given_different_columns_then_return_both(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_ANOTHER_COLUMN, _A_TARGET).build()]} + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_ANOTHER_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) @@ -58,9 +98,23 @@ def test_given_different_columns_then_return_both(self) -> None: ] } - def test_given_same_column_but_different_value_then_prioritize_confirmed(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _ANOTHER_TARGET).build()]} + def test_given_same_column_but_different_value_then_prioritize_confirmed( + self, + ) -> None: + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _ANOTHER_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py index 50a0209655cb..303799e245eb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py @@ -13,5 +13,4 @@ class BaseBackend(ABC): """ @abstractmethod - def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: - ... + def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: ... diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py index cd6d61ee5d6c..204f1c099fde 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py @@ -9,6 +9,7 @@ import duckdb from airbyte_protocol.models import AirbyteMessage # type: ignore + from live_tests.commons.backends.file_backend import FileBackend diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py index a4d0b57c910a..ae3e68c228f8 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py @@ -10,6 +10,7 @@ from airbyte_protocol.models import AirbyteMessage # type: ignore from airbyte_protocol.models import Type as AirbyteMessageType from cachetools import LRUCache, cached + from live_tests.commons.backends.base_backend import BaseBackend from live_tests.commons.utils import sanitize_stream_name @@ -123,7 +124,11 @@ def _get_filepaths_and_messages(self, message: AirbyteMessage) -> tuple[tuple[st stream_file_path_data_only = self.record_per_stream_directory / f"{sanitize_stream_name(stream_name)}_data_only.jsonl" self.record_per_stream_paths[stream_name] = stream_file_path self.record_per_stream_paths_data_only[stream_name] = stream_file_path_data_only - return (self.RELATIVE_RECORDS_PATH, str(stream_file_path), str(stream_file_path_data_only),), ( + return ( + self.RELATIVE_RECORDS_PATH, + str(stream_file_path), + str(stream_file_path_data_only), + ), ( message.json(sort_keys=True), message.json(sort_keys=True), json.dumps(message.record.data, sort_keys=True), diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py index adcb1c6868cb..5e2172acf398 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py @@ -10,6 +10,7 @@ import rich from connection_retriever import ConnectionObject, retrieve_objects # type: ignore from connection_retriever.errors import NotPermittedError # type: ignore + from live_tests.commons.models import ConnectionSubset from live_tests.commons.utils import build_connection_url diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py index 9f32635ce181..5b2e5efe1675 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py @@ -15,6 +15,7 @@ import anyio import asyncer import dagger + from live_tests.commons import errors from live_tests.commons.models import Command, ExecutionInputs, ExecutionResult from live_tests.commons.proxy import Proxy diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index f762a2f94ad2..d58e88c04026 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import _collections_abc import json import logging import tempfile @@ -13,17 +14,21 @@ from pathlib import Path from typing import Any, Dict, List, Optional -import _collections_abc import dagger import requests -from airbyte_protocol.models import AirbyteCatalog # type: ignore -from airbyte_protocol.models import AirbyteMessage # type: ignore -from airbyte_protocol.models import AirbyteStateMessage # type: ignore -from airbyte_protocol.models import AirbyteStreamStatusTraceMessage # type: ignore -from airbyte_protocol.models import ConfiguredAirbyteCatalog # type: ignore -from airbyte_protocol.models import TraceType # type: ignore +from airbyte_protocol.models import ( + AirbyteCatalog, # type: ignore + AirbyteMessage, # type: ignore + AirbyteStateMessage, # type: ignore + AirbyteStreamStatusTraceMessage, # type: ignore + ConfiguredAirbyteCatalog, # type: ignore + TraceType, # type: ignore +) from airbyte_protocol.models import Type as AirbyteMessageType from genson import SchemaBuilder # type: ignore +from mitmproxy import http +from pydantic import ValidationError + from live_tests.commons.backends import DuckDbBackend, FileBackend from live_tests.commons.secret_access import get_airbyte_api_key from live_tests.commons.utils import ( @@ -33,8 +38,6 @@ sanitize_stream_name, sort_dict_keys, ) -from mitmproxy import http -from pydantic import ValidationError class UserDict(_collections_abc.MutableMapping): # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py index a9e2f6cd54fd..6b11b405c9cc 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py @@ -15,6 +15,8 @@ from airbyte_protocol.models import AirbyteCatalog, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification # type: ignore from connection_retriever.audit_logging import get_user_email # type: ignore from connection_retriever.retrieval import ConnectionNotFoundError, NotPermittedError, get_current_docker_image_tag # type: ignore +from rich.prompt import Confirm, Prompt + from live_tests import stash_keys from live_tests.commons.connection_objects_retrieval import ConnectionObject, InvalidConnectionError, get_connection_objects from live_tests.commons.connector_runner import ConnectorRunner, Proxy @@ -35,7 +37,6 @@ from live_tests.commons.utils import build_connection_url, clean_up_artifacts from live_tests.report import Report, ReportState from live_tests.utils import get_catalog, get_spec -from rich.prompt import Confirm, Prompt if TYPE_CHECKING: from _pytest.config import Config diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py index 443ab2f1ac4d..7467fed152b7 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py @@ -5,6 +5,7 @@ import pytest from airbyte_protocol.models import Status, Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py index 61fac4147986..56f955383dbf 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py @@ -7,6 +7,7 @@ import pytest from _pytest.fixtures import SubRequest from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_catalog diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py index 4530d7008660..3587bb586a23 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py @@ -8,6 +8,7 @@ import pytest from airbyte_protocol.models import AirbyteMessage # type: ignore from deepdiff import DeepDiff # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_test_logger, write_string_to_test_artifact diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py index 967698f7462a..7a1985fd3e40 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py @@ -5,6 +5,7 @@ import pytest from airbyte_protocol.models import Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/report.py b/airbyte-ci/connectors/live-tests/src/live_tests/report.py index 255843f66143..4320d164619a 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/report.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/report.py @@ -14,6 +14,7 @@ import requests import yaml from jinja2 import Environment, PackageLoader, select_autoescape + from live_tests import stash_keys from live_tests.consts import MAX_LINES_IN_REPORT @@ -21,6 +22,7 @@ import pytest from _pytest.config import Config from airbyte_protocol.models import SyncMode, Type # type: ignore + from live_tests.commons.models import Command, ExecutionResult diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py b/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py index 493182e264d0..f93488c214bf 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest + from live_tests.commons.evaluation_modes import TestEvaluationMode from live_tests.commons.models import ConnectionObjects, ConnectionSubset from live_tests.report import Report diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/utils.py index 8c8e039ebbb5..3db571751cc4 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/utils.py @@ -11,11 +11,12 @@ import pytest from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, ConnectorSpecification, Status, Type # type: ignore from deepdiff import DeepDiff # type: ignore +from mitmproxy import http, io # type: ignore +from mitmproxy.addons.savehar import SaveHar # type: ignore + from live_tests import stash_keys from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT -from mitmproxy import http, io # type: ignore -from mitmproxy.addons.savehar import SaveHar # type: ignore if TYPE_CHECKING: from _pytest.fixtures import SubRequest diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py index fc5fd2d7813b..ac945b830ffb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py @@ -7,6 +7,7 @@ import pytest from airbyte_protocol.models import Type + from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py index 44538edaded2..73e4975acbd7 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py @@ -9,6 +9,7 @@ import jsonschema import pytest from airbyte_protocol.models import AirbyteCatalog + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py index 16f176eb4e4e..5d04f883960f 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py @@ -16,6 +16,7 @@ AirbyteStreamStatusTraceMessage, ConfiguredAirbyteCatalog, ) + from live_tests.commons.json_schema_helper import conforms_to_schema from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_test_logger diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py index e1619023b63f..ae573019bf2a 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py @@ -9,6 +9,7 @@ import jsonschema import pytest from airbyte_protocol.models import ConnectorSpecification + from live_tests.commons.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_paths_in_connector_config from live_tests.commons.models import ExecutionResult, SecretDict from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema, get_test_logger diff --git a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py index be22da351d93..fbd8f03bc4f7 100644 --- a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py +++ b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py @@ -13,6 +13,7 @@ Status, ) from airbyte_protocol.models import Type as AirbyteMessageType + from live_tests.commons.backends import FileBackend diff --git a/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py b/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py index b9fec6c28336..6b73fbfa3ceb 100644 --- a/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py +++ b/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py @@ -16,6 +16,8 @@ SyncMode, Type, ) +from pydantic import BaseModel + from live_tests.commons.json_schema_helper import ( ComparableType, JsonSchemaHelper, @@ -23,7 +25,6 @@ get_expected_schema_structure, get_object_structure, ) -from pydantic import BaseModel def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py index 795702333793..879bc8459ca7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py @@ -5,6 +5,8 @@ import pathlib import click +from pydantic import ValidationError + from metadata_service.constants import METADATA_FILE_NAME from metadata_service.gcs_upload import ( MetadataDeleteInfo, @@ -14,7 +16,6 @@ upload_metadata_to_gcs, ) from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load -from pydantic import ValidationError def log_metadata_upload_info(metadata_upload_info: MetadataUploadInfo): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py index 4775f25cdd9a..7cd3d94a3d1c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py @@ -16,6 +16,9 @@ import yaml from google.cloud import storage from google.oauth2 import service_account +from pydash import set_ +from pydash.objects import get + from metadata_service.constants import ( COMPONENTS_PY_FILE_NAME, COMPONENTS_ZIP_FILE_NAME, @@ -34,8 +37,6 @@ from metadata_service.models.generated.GitInfo import GitInfo from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load -from pydash import set_ -from pydash.objects import get # 🧩 TYPES diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py index b23578061953..3866a276bf7c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py @@ -8,11 +8,12 @@ import semver import yaml -from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from pydantic import ValidationError from pydash.objects import get +from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 + @dataclass(frozen=True) class ValidatorOptions: @@ -247,7 +248,6 @@ def validate_rc_suffix_and_rollout_configuration( if docker_image_tag is None: return False, "The dockerImageTag field is not set." try: - is_major_release_candidate_version = check_is_major_release_candidate_version(docker_image_tag) is_dev_version = check_is_dev_version(docker_image_tag) is_rc_version = check_is_release_candidate_version(docker_image_tag) diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py index 4a821bb31870..e57054eb879a 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py @@ -6,11 +6,12 @@ import pytest from click.testing import CliRunner +from pydantic import BaseModel, ValidationError, error_wrappers +from test_gcs_upload import stub_is_image_on_docker_hub + from metadata_service import commands from metadata_service.gcs_upload import MetadataUploadInfo, UploadedFile from metadata_service.validators.metadata_validator import ValidatorOptions, validate_docker_image_tag_is_not_decremented -from pydantic import BaseModel, ValidationError, error_wrappers -from test_gcs_upload import stub_is_image_on_docker_hub NOT_TEST_VALIDATORS = [ # Not testing validate_docker_image_tag_is_not_decremented as its tested independently in test_validators @@ -19,6 +20,7 @@ PATCHED_VALIDATORS = [v for v in commands.PRE_UPLOAD_VALIDATORS if v not in NOT_TEST_VALIDATORS] + # TEST VALIDATE COMMAND def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path): runner = CliRunner() diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py index 58a21c52ae2f..e5e30549d116 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py @@ -6,6 +6,7 @@ import warnings import pytest + from metadata_service import docker_hub diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py index f3f9f7c0fae6..6e8cf8abc66d 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py @@ -7,6 +7,8 @@ import pytest import yaml +from pydash.objects import get + from metadata_service import gcs_upload from metadata_service.constants import ( COMPONENTS_PY_FILE_NAME, @@ -19,7 +21,6 @@ from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import ValidatorOptions -from pydash.objects import get MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["99.99.99", "0.0.0"] MISSING_SHA = "MISSINGSHA" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py index 9ce15092fcb7..eac4bfd2cde2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py @@ -5,15 +5,16 @@ from unittest.mock import patch import pytest + from metadata_service.spec_cache import CachedSpec, Registries, SpecCache, get_docker_info_from_spec_cache_path @pytest.fixture def mock_spec_cache(): - with patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, patch( - "google.cloud.storage.Client.bucket" - ) as MockBucket: - + with ( + patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, + patch("google.cloud.storage.Client.bucket") as MockBucket, + ): # Create stub mock client and bucket MockClient.return_value MockBucket.return_value diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py index cc87c86e7af0..2d39585cb1c5 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py @@ -5,6 +5,7 @@ import pathlib import yaml + from metadata_service.models import transform from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py index 4946f6964484..b4ac1d44faad 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py @@ -4,6 +4,7 @@ import requests import semver import yaml + from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.validators import metadata_validator diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py index 20afa5a6b391..0948c73bdf39 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py @@ -11,6 +11,7 @@ from google.cloud import storage from orchestrator.logging import sentry + GROUP_NAME = "connector_metrics" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py index 6d2ca249f076..1fe1ada3c5f1 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py @@ -22,6 +22,7 @@ ) from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + T = TypeVar("T") GROUP_NAME = "connector_test_report" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py index 63f939244223..9d1d875feb18 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py @@ -17,6 +17,7 @@ from orchestrator.ops.slack import send_slack_message from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "github" TOOLING_TEAM_SLACK_TEAM_ID = "S077R8636CV" # We give 6 hours for the metadata to be updated diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py index ea3d0fb54a7d..783f16614b44 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py @@ -16,6 +16,7 @@ from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition from orchestrator.utils.object_helpers import are_values_equal, merge_values + GROUP_NAME = "metadata" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py index dde8232c8a69..dc00e6412c8d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py @@ -20,6 +20,7 @@ from orchestrator.utils.object_helpers import default_none_to_dict from pydash.objects import set_with + PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] GROUP_NAME = "registry" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py index 657ef41ce5af..d642f5179bda 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py @@ -31,6 +31,7 @@ from pydantic import BaseModel, ValidationError from pydash.objects import get, set_with + GROUP_NAME = "registry_entry" # TYPES diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py index a1e26af19329..a1fa7a6e198d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py @@ -22,6 +22,7 @@ ) from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "registry_reports" OSS_SUFFIX = "_oss" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py index 8bfae8867751..2bbcbd283451 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py @@ -8,6 +8,7 @@ from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, Output, asset from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "slack" USER_REQUEST_CHUNK_SIZE = 2000 diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py index a82cbc5df355..3d52193b1b95 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py @@ -11,6 +11,7 @@ from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from orchestrator.logging import sentry + GROUP_NAME = "specs_secrets_mask" # HELPERS diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py index e94fa49d225c..b7f803334efe 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py @@ -5,6 +5,7 @@ import os from typing import Optional + DEFAULT_ASSET_URL = "https://storage.googleapis.com" VALID_REGISTRIES = ["oss", "cloud"] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py index 331ccab653d3..904963eeed03 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py @@ -7,6 +7,7 @@ import requests from orchestrator.models.metadata import LatestMetadataEntry + GROUP_NAME = "connector_cdk_versions" BASE_URL = "https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py index 7021e5a7d991..f32a0a39fde4 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py @@ -10,6 +10,7 @@ from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition + PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py index 4528a6d8c31f..dd3eb607b43e 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py @@ -4,6 +4,7 @@ from dagster import AssetSelection, define_asset_job + nightly_reports_inclusive = AssetSelection.keys("generate_nightly_report").upstream() generate_nightly_reports = define_asset_job(name="generate_nightly_reports", selection=nightly_reports_inclusive) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py index 01ce46dace71..275f27e25813 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py @@ -4,6 +4,7 @@ from dagster import AssetSelection, define_asset_job + stale_gcs_latest_metadata_file_inclusive = AssetSelection.keys("stale_gcs_latest_metadata_file").upstream() generate_stale_gcs_latest_metadata_file = define_asset_job( name="generate_stale_metadata_report", selection=stale_gcs_latest_metadata_file_inclusive diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py index 322c2e5d3002..3756f7ac8a87 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py @@ -7,6 +7,7 @@ from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus + oss_registry_inclusive = AssetSelection.keys("persisted_oss_registry", "specs_secrets_mask_yaml").upstream() generate_oss_registry = define_asset_job(name="generate_oss_registry", selection=oss_registry_inclusive) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py index e65d9fd691e8..8e908183e80b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py @@ -8,6 +8,7 @@ import sentry_sdk from dagster import AssetExecutionContext, OpExecutionContext, SensorEvaluationContext, get_dagster_logger + sentry_logger = get_dagster_logger("sentry") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py index a8e791abd5bb..1f956e619ead 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Extra + # TODO (ben): When the pipeline project is brought into the airbyte-ci folder # we should update these models to import their twin models from the pipeline project diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py index 78477fe58639..9eac1b19ebf1 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py @@ -13,6 +13,7 @@ from dagster._utils import mkdir_p from typing_extensions import TypeAlias + IOStream: TypeAlias = Union[TextIO, BinaryIO] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py index cc3a3d42f402..abbb2aef30f6 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py @@ -6,6 +6,7 @@ from dagster import DefaultSensorStatus, RunRequest, SensorDefinition, SensorEvaluationContext, SkipReason, build_resources, sensor + # e.g. 2023-06-02T17:42:36Z EXPECTED_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py index eaa76f7b30c7..1fcbade3c6a4 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py @@ -12,6 +12,7 @@ from jinja2 import Environment, PackageLoader from orchestrator.utils.object_helpers import deep_copy_params + # 🔗 HTML Renderers diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py index 64b482884b69..b5c996fe26c0 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py @@ -8,6 +8,7 @@ import pandas as pd from dagster import MetadataValue, Output + OutputDataFrame = Output[pd.DataFrame] CURSOR_SEPARATOR = ":" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py index 19b57416297a..77ebca0a4ee3 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py @@ -9,6 +9,7 @@ import mergedeep from deepdiff import DeepDiff + T = TypeVar("T") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py index 27088881908e..b0a7f223af3a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger + from pipelines import main_logger from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py index e26dc095a7c0..a6565dfe5495 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py @@ -11,6 +11,7 @@ from click import UsageError from connector_ops.utils import Connector # type: ignore from dagger import Container, ExecError, Platform, QueryError + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.utils import export_container_to_tarball, sh_dash_c from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py index 8d31bd5a714a..70bcdfaf1837 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py @@ -3,6 +3,7 @@ # from dagger import Container, Directory, File, Platform, QueryError + from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.steps.gradle import GradleTask diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py index 0c9116d592da..72c9dc386f1e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py @@ -6,13 +6,14 @@ from typing import Any from dagger import Container, Platform +from pydash.objects import get # type: ignore + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import COMPONENTS_FILE_PATH, MANIFEST_FILE_PATH from pipelines.dagger.actions.python.common import apply_python_development_overrides from pipelines.models.steps import StepResult -from pydash.objects import get # type: ignore class BuildConnectorImages(BuildConnectorImagesBase): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py index 3eaa9e9b088d..2cfe294a0cac 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py @@ -3,6 +3,7 @@ # from dagger import Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.connector import normalization from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py index 262fdbc780d3..802eae40cca0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -6,6 +6,7 @@ from typing import Any from dagger import Container, Platform + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py index 3c7ab35dcff5..22dee605124b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import semver + from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index ca69acb7a4a0..6de4dc1e0260 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -8,6 +8,7 @@ import asyncclick as click from connector_ops.utils import Connector, ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore + from pipelines import main_logger from pipelines.cli.airbyte_ci import wrap_in_secret from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 52deb09d7c01..ecb901945486 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -15,6 +15,7 @@ from asyncer import asyncify from dagger import Directory, Platform from github import PullRequest + from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import BUILD_PLATFORMS from pipelines.dagger.actions import secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py index 156b7ab732a2..6203febb51b9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py @@ -4,6 +4,7 @@ from typing import List import asyncclick as click + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.generate_erd.pipeline import run_connector_generate_erd_pipeline from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py index 54b0cd4b8e5e..17b70bd2475c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, List from dagger import Container, Directory + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext @@ -51,7 +52,7 @@ async def _run(self, connector_to_discover: Container) -> StepResult: command.append("--skip-llm-relationships") erd_directory = self._build_erd_container(connector_directory, discovered_catalog).with_exec(command).directory("/source/erd") - await (erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory)))) + await erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory))) return StepResult(step=self, status=StepStatus.SUCCESS, output=erd_directory) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 72bfadc2cc20..dea261788ba5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -7,10 +7,11 @@ import asyncclick as click from connector_ops.utils import console # type: ignore -from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from rich.table import Table from rich.text import Text +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + @click.command(cls=DaggerPipelineCommand, help="List all selected connectors.", name="list") @click.option( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py index 4c933ce975ec..d87831d84535 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_base_image.pipeline import run_connector_migration_to_base_image_pipeline from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py index 62d62afcb717..57ea1b10d8cf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py @@ -8,6 +8,7 @@ from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Directory from jinja2 import Template + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata @@ -15,7 +16,6 @@ from pipelines.models.steps import Step, StepResult, StepStatus if TYPE_CHECKING: - from anyio import Semaphore @@ -89,7 +89,6 @@ async def _run(self) -> StepResult: ) def add_build_instructions(self, og_doc_content: str) -> str: - build_instructions_template = Template( textwrap.dedent( """ diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py index 2c5070adc84e..da73b3ebc022 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.migrate_to_inline_schemas.pipeline import run_connector_migrate_to_inline_schemas_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py index ccd85b81adbd..ba880ae73840 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, List from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines import main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py index 2270baa15b3d..9d32a6e97614 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.migrate_to_logging_logger.pipeline import run_connector_migrate_to_logging_logger_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py index 1e8d5ce8500a..d79eedaaa7c3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py index 07ac847dd7d3..f7440de7479a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_manifest_only.pipeline import run_connectors_manifest_only_pipeline from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines @@ -12,7 +13,6 @@ @click.command(cls=DaggerPipelineCommand, short_help="Migrate a low-code connector to manifest-only") @click.pass_context async def migrate_to_manifest_only(ctx: click.Context) -> bool: - connectors_contexts = [ ConnectorContext( pipeline_name=f"Migrate connector {connector.technical_name} to manifest-only", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py index e32946d26087..89e167de379e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py @@ -10,6 +10,7 @@ import git # type: ignore from anyio import Semaphore # type: ignore from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_manifest_only.manifest_component_transformer import ManifestComponentTransformer @@ -297,7 +298,6 @@ async def _run(self) -> StepResult: ## MAIN FUNCTION ## async def run_connectors_manifest_only_pipeline(context: ConnectorContext, semaphore: "Semaphore", *args: Any) -> Report: - steps_to_run: STEP_TREE = [] steps_to_run.append([StepToRun(id=CONNECTOR_TEST_STEP_ID.MANIFEST_ONLY_CHECK, step=CheckIsManifestMigrationCandidate(context))]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py index 4e73eda537a0..eef7b8c9e1cf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_poetry.pipeline import run_connector_migration_to_poetry_pipeline_wrapper from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines @@ -31,7 +32,6 @@ ) @click.pass_context async def migrate_to_poetry(ctx: click.Context, changelog: bool, bump: str | None) -> bool: - connectors_contexts = [ ConnectorContext( pipeline_name=f"Migrate {connector.technical_name} to Poetry", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py index ccb576c47c9d..f29bfb213fff 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py @@ -12,6 +12,7 @@ import toml from connector_ops.utils import ConnectorLanguage # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py index 4e6f03512e18..2ca0f8c55b08 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py @@ -3,6 +3,7 @@ # """This module groups the functions to run full pipelines for connector testing.""" + from __future__ import annotations import sys @@ -13,6 +14,7 @@ import dagger from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Config + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 5a93fa05b2f9..67df171d49b3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, Iterable, List import asyncclick as click + from pipelines import main_logger from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext, RolloutMode diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index f4e3e132eccc..702461da73dc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -9,6 +9,7 @@ import asyncclick as click from github import PullRequest + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import PUBLISH_FAILURE_SLACK_CHANNEL, PUBLISH_UPDATES_SLACK_CHANNEL, ContextState from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles @@ -136,7 +137,6 @@ def get_slack_channels(self) -> List[str]: return [PUBLISH_UPDATES_SLACK_CHANNEL] def create_slack_message(self) -> str: - docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" message = f"*{self.rollout_mode.value} <{docker_hub_url}|{self.docker_image}>*\n" if self.is_ci: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 56af45b38413..4ced8a931b46 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -15,6 +15,8 @@ from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification # type: ignore from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage # type: ignore from dagger import Container, Directory, ExecError, File, ImageLayerCompression, Platform, QueryError +from pydantic import BaseModel, ValidationError + from pipelines import consts from pipelines.airbyte_ci.connectors.build_image import steps from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext, RolloutMode @@ -30,7 +32,6 @@ from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file from pipelines.helpers.pip import is_package_published from pipelines.models.steps import Step, StepModifyingFiles, StepResult, StepStatus -from pydantic import BaseModel, ValidationError class InvalidSpecOutputError(Exception): @@ -244,8 +245,10 @@ async def check_if_image_only_has_gzip_layers(self) -> bool: async def _run(self, attempt: int = 3) -> StepResult: try: try: - await self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image}").with_exec( - ["spec"], use_entrypoint=True + await ( + self.context.dagger_client.container() + .from_(f"docker.io/{self.context.docker_image}") + .with_exec(["spec"], use_entrypoint=True) ) except ExecError: if attempt > 0: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py index daf63427e58b..6237ffdc1786 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.pull_request.pipeline import run_connector_pull_request_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 68f75275ad03..1447a10fcee8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -12,24 +12,26 @@ from connector_ops.utils import console # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape +from rich.console import Group +from rich.panel import Panel +from rich.style import Style +from rich.table import Table +from rich.text import Text + from pipelines.consts import GCS_PUBLIC_DOMAIN from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX, AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX from pipelines.helpers.utils import format_duration from pipelines.models.artifacts import Artifact from pipelines.models.reports import Report from pipelines.models.steps import StepStatus -from rich.console import Group -from rich.panel import Panel -from rich.style import Style -from rich.table import Table -from rich.text import Text if TYPE_CHECKING: from typing import List - from pipelines.airbyte_ci.connectors.context import ConnectorContext from rich.tree import RenderableType + from pipelines.airbyte_ci.connectors.context import ConnectorContext + @dataclass(frozen=True) class ConnectorReport(Report): @@ -133,9 +135,9 @@ def to_html(self) -> str: template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url - template_context[ - "icon_url" - ] = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + template_context["icon_url"] = ( + f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + ) return template.render(template_context) async def save_html_report(self) -> None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 442a5a824245..7a4b1179edd9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -6,6 +6,7 @@ from typing import Dict, List import asyncclick as click + from pipelines import main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py index 2b94b34e7a50..b941d05f034e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py @@ -8,11 +8,12 @@ from logging import Logger from typing import Any, Dict, List, Optional +from pydash import find # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.execution.run_steps import RunStepOptions from pipelines.models.secrets import Secret, SecretNotFoundError, SecretStore -from pydash import find # type: ignore # These test suite names are declared in metadata.yaml files TEST_SUITE_NAME_TO_STEP_ID = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py index 16533aa192dd..495bfc17d570 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -9,6 +9,7 @@ import anyio from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext @@ -17,7 +18,6 @@ from pipelines.helpers.execution.run_steps import StepToRun, run_steps if TYPE_CHECKING: - from pipelines.helpers.execution.run_steps import STEP_TREE LANGUAGE_MAPPING = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index afecc5fe00ee..7fbd7c8a063f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -20,6 +20,11 @@ import semver import yaml # type: ignore from dagger import Container, Directory + +# This slugify lib has to be consistent with the slugify lib used in live_tests +# live_test can't resolve the passed connector container otherwise. +from slugify import slugify # type: ignore + from pipelines import hacks, main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -33,10 +38,6 @@ from pipelines.models.secrets import Secret from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus -# This slugify lib has to be consistent with the slugify lib used in live_tests -# live_test can't resolve the passed connector container otherwise. -from slugify import slugify # type: ignore - GITHUB_URL_PREFIX_FOR_CONNECTORS = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/master/airbyte-integrations/connectors" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py index 2cf011a19d65..caef57836570 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py @@ -3,12 +3,14 @@ # """This module groups steps made to run tests for a specific Java connector given a test context.""" + from __future__ import annotations from typing import TYPE_CHECKING import anyio from dagger import File, QueryError + from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import ( BuildConnectorDistributionTar, BuildConnectorImages, @@ -89,7 +91,6 @@ def _create_integration_step_args_factory(context: ConnectorTestContext) -> Call """ async def _create_integration_step_args(results: RESULTS_DICT) -> Dict[str, Optional[File]]: - connector_container = results["build"].output[LOCAL_BUILD_PLATFORM] connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container, LOCAL_BUILD_PLATFORM) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py index c41ed2652a28..00a5eca6be57 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py @@ -7,6 +7,7 @@ from typing import List, Sequence, Tuple from dagger import Container, File + from pipelines.airbyte_ci.connectors.build_image.steps.manifest_only_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 5f38935d67e4..a967bcaefdf1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -8,9 +8,10 @@ from typing import List, Sequence, Tuple import dpath.util +from dagger import Container, File + import pipelines.dagger.actions.python.common import pipelines.dagger.actions.system.docker -from dagger import Container, File from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py index 5bce0e3d597d..03111abf2ef1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py @@ -5,6 +5,7 @@ from typing import List import asyncclick as click + from pipelines.airbyte_ci.connectors.up_to_date.pipeline import run_connector_up_to_date_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py index bd9dd2b6ea95..b8fea724e5e7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING from jinja2 import Environment, PackageLoader, select_autoescape + from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -25,6 +26,7 @@ from anyio import Semaphore from github import PullRequest + from pipelines.models.steps import StepResult UP_TO_DATE_PR_LABEL = "up-to-date" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py index 457b964849ec..f461fb8d0c88 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py @@ -14,6 +14,7 @@ import dagger from connector_ops.utils import POETRY_LOCK_FILE_NAME, PYPROJECT_FILE_NAME # type: ignore from deepdiff import DeepDiff # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.models.steps import Step, StepModifyingFiles, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py index 35ce460599bf..16519b4e6d6e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py index 623624f2f296..a764dcef2bcc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py @@ -10,6 +10,7 @@ import toml from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import LOCAL_BUILD_PLATFORM diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index a1f2d3cc613f..c1e569392db1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -5,6 +5,7 @@ """ Module exposing the format commands. """ + from __future__ import annotations import logging @@ -12,6 +13,7 @@ from typing import Dict, List import asyncclick as click + from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index b18464b1539f..447acc8be71b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Union import dagger + from pipelines.airbyte_ci.format.consts import CACHE_MOUNT_PATH, DEFAULT_FORMAT_IGNORE_LIST, REPO_MOUNT_PATH, WARM_UP_INCLUSIONS, Formatter from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME, GO_IMAGE, MAVEN_IMAGE, NODE_IMAGE, PYTHON_3_10_IMAGE from pipelines.helpers import cache_keys diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py index da6e4054270f..195eb7639ec5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py @@ -9,6 +9,7 @@ import asyncclick as click import dagger + from pipelines import main_logger from pipelines.airbyte_ci.format.actions import list_files_in_directory from pipelines.airbyte_ci.format.configuration import Formatter diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py index a140b322309f..98b801eb8e27 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py index de33dfcb254a..f8a4d490ff20 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py @@ -8,6 +8,7 @@ import asyncclick as click import dagger + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.steps.docker import SimpleDockerStep diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 72dbe53b170f..e40905692072 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -5,9 +5,11 @@ """ Module exposing the format commands. """ + from __future__ import annotations import asyncclick as click + from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index e02c853a4275..d822ae8e890b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -5,12 +5,14 @@ """ Module exposing the format commands. """ + from __future__ import annotations from typing import Optional import asyncclick as click from packaging import version + from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py index 6a2fcc020b32..210cd2f8bc5f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py @@ -7,6 +7,7 @@ import yaml from base_images import version_registry # type: ignore from connector_ops.utils import METADATA_FILE_NAME # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file from pipelines.models.steps import StepModifyingFiles, StepResult, StepStatus @@ -22,7 +23,6 @@ class NoBaseImageAddressInMetadataError(Exception): class UpdateBaseImageMetadata(StepModifyingFiles): - BASE_IMAGE_LIST_CACHE_TTL_SECONDS = 60 * 60 * 24 # 1 day context: ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py index 3bf22b823a95..e26c10c292c1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py @@ -9,6 +9,7 @@ import semver import yaml # type: ignore from connector_ops.utils import METADATA_FILE_NAME, PYPROJECT_FILE_NAME # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py index 96386bf0f537..3170d082b333 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py @@ -8,6 +8,7 @@ import semver from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.changelog import Changelog from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file @@ -33,7 +34,6 @@ def __init__( self.pull_request_number = pull_request_number or "*PR_NUMBER_PLACEHOLDER*" async def _run(self, pull_request_number: int | str | None = None) -> StepResult: - if pull_request_number is None: # this allows passing it dynamically from a result of another action (like creating a pull request) pull_request_number = self.pull_request_number diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py index 191687f0ac19..1e974fc46deb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional import dagger + from pipelines.dagger.actions.python.pipx import with_installed_pipx_package from pipelines.dagger.containers.python import with_python_base from pipelines.models.contexts.pipeline_context import PipelineContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 0cc44ad511af..8dfdb42bbbd7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -6,9 +6,10 @@ from datetime import datetime from typing import Any, ClassVar, List, Optional, Tuple, cast -import pipelines.dagger.actions.system.docker import requests from dagger import CacheSharingMode, CacheVolume, Container, ExecError + +import pipelines.dagger.actions.system.docker from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME, AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py index 17c5a3312c08..65c31518ccf1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -11,6 +11,7 @@ import tomli import tomli_w from dagger import Container, Directory + from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.utils import sh_dash_c diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 117a1cf5e0d0..15ec6831423b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import asyncer + from pipelines.airbyte_ci.test import INTERNAL_POETRY_PACKAGES, INTERNAL_POETRY_PACKAGES_PATH, pipeline from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.git import get_modified_files diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py index 7f1670636ccb..3b7b758b5120 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py @@ -9,6 +9,7 @@ import asyncer import dagger import toml + from pipelines.airbyte_ci.test.models import deserialize_airbyte_ci_config from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_VERSION, POETRY_CACHE_VOLUME_NAME, PYPROJECT_TOML_FILE_PATH from pipelines.dagger.actions.system import docker diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py index c633f59db1d2..690c5ae56f94 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py @@ -5,6 +5,7 @@ import logging import asyncclick as click + from pipelines.cli.auto_update import is_dev_command from pipelines.external_scripts.airbyte_ci_dev_install import main as install_airbyte_ci_dev_pipx from pipelines.external_scripts.airbyte_ci_install import main as install_airbyte_ci_binary diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index fb7222ed1929..a9335d2cab3f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -22,6 +22,7 @@ import asyncclick as click import docker # type: ignore from github import PullRequest + from pipelines import main_logger from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag from pipelines.cli.click_decorators import ( diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py index a97291aafaf9..6f7c16bae0ca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py @@ -13,6 +13,7 @@ import asyncclick as click import requests + from pipelines import main_logger from pipelines.cli.confirm_prompt import confirm from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index b88f582c6e37..7ede57342b7b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -8,6 +8,7 @@ from typing import Any, Callable, Type, TypeVar import asyncclick as click + from pipelines.models.ci_requirements import CIRequirements _AnyCallable = Callable[..., Any] diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py index dfc0bd9d626f..9120bf59921e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py @@ -3,6 +3,7 @@ # """This module groups util function used in pipelines.""" + from __future__ import annotations import sys @@ -10,6 +11,7 @@ import asyncclick as click from dagger import DaggerError + from pipelines import consts, main_logger from pipelines.consts import GCS_PUBLIC_DOMAIN, STATIC_REPORT_PREFIX from pipelines.helpers import sentry_utils diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py index 5233cd62bef8..184d4ada00d6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -14,6 +14,7 @@ import pkg_resources # type: ignore import requests # type: ignore + from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME LOGGER = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py index 84923558d3d1..a477e89d2a71 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py @@ -3,6 +3,7 @@ from typing import Any, Optional import asyncclick as click + from pipelines.helpers.gcs import sanitize_gcp_credentials from pipelines.models.secrets import InMemorySecretStore, Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py index 4223f86b26ec..c9e506813c04 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py @@ -7,6 +7,7 @@ from importlib.abc import Loader from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py index 9fe2806b7e27..406293ab9437 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py @@ -5,6 +5,7 @@ from typing import Any, Dict from dagger import Container, Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 22e11f22619b..5e5a3808b7b3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -8,6 +8,7 @@ from click import UsageError from dagger import Container, Directory + from pipelines import hacks from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import PATH_TO_LOCAL_CDK diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py index 3c16cb043520..aa8b7cd90654 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py @@ -5,6 +5,7 @@ from typing import List, Optional from dagger import Container + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package from pipelines.dagger.actions.python.poetry import find_local_dependencies_in_pyproject_toml diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py index 09bf5f683462..cc87e2506734 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py @@ -8,6 +8,7 @@ import toml from dagger import Container, Directory + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py index b7f9095b9592..f8f8f28accf7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py @@ -9,6 +9,7 @@ from typing import List, Optional, Tuple from dagger import Client, File + from pipelines.helpers.utils import get_exec_result, secret_host_variable, with_exit_code from pipelines.models.secrets import Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index 81773fd7ea4a..606adce68c8f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -3,6 +3,7 @@ # """This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory.""" + from __future__ import annotations from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ from typing import Callable, List from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py index 61ec8c401106..72f1827c9939 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py @@ -6,9 +6,9 @@ import uuid from typing import Callable, Dict, List, Optional, Union -from dagger import Client, Container, File +from dagger import Client, Container, File, Service from dagger import Secret as DaggerSecret -from dagger import Service + from pipelines import consts from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import ( @@ -229,9 +229,8 @@ def with_crane( if context.docker_hub_username and context.docker_hub_password: base_container = ( - base_container.with_secret_variable( - "DOCKER_HUB_USERNAME", context.docker_hub_username.as_dagger_secret(context.dagger_client) - ).with_secret_variable("DOCKER_HUB_PASSWORD", context.docker_hub_password.as_dagger_secret(context.dagger_client)) + base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username.as_dagger_secret(context.dagger_client)) + .with_secret_variable("DOCKER_HUB_PASSWORD", context.docker_hub_password.as_dagger_secret(context.dagger_client)) # We use sh -c to be able to use environment variables in the command # This is a workaround as the default crane entrypoint doesn't support environment variables .with_exec(sh_dash_c(["crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"])) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py index cf38d65be91b..790063e2ce66 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py @@ -4,6 +4,7 @@ from typing import Optional from dagger import Client, Container + from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL from pipelines.helpers.utils import sh_dash_c diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py index 2eb424189439..8438fcec56e1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py @@ -3,6 +3,7 @@ # from dagger import Container, Secret + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import INTERNAL_TOOL_PATHS from pipelines.dagger.actions.python.pipx import with_installed_pipx_package diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index 47bbe7822b21..9c595db41f63 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -5,6 +5,7 @@ import datetime from dagger import CacheVolume, Container, File, Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions.connector.hooks import finalize_build diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 245190f66ede..7bf7afc8521e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -4,6 +4,7 @@ from dagger import CacheSharingMode, CacheVolume, Client, Container + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import ( CONNECTOR_TESTING_REQUIREMENTS, diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 76c968236947..28b7b298c4ae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -11,12 +11,14 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines import consts from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, is_automerge_pull_request, update_commit_status_check if TYPE_CHECKING: from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.models.steps import StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py index 41a87b27cd11..a062b185f767 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py @@ -9,6 +9,7 @@ from typing import Set, Tuple import semver + from pipelines.helpers.github import AIRBYTE_GITHUB_REPO diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index 4f601b7e83dc..4440fd4e466c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -10,6 +10,7 @@ import asyncclick as click import asyncer from jinja2 import Template + from pipelines.models.steps import CommandResult ALL_RESULTS_KEY = "_run_all_results" diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py index ad9531e90702..2f822c0e8f2b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Callable, List import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py index 98c2d95bb69c..a6e8102bdf35 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py @@ -7,6 +7,7 @@ from typing import FrozenSet, Set, Union from connector_ops.utils import Connector # type: ignore + from pipelines import main_logger from pipelines.helpers.utils import IGNORED_FILE_EXTENSIONS, METADATA_FILE_NAME diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py index 7330f99039fc..2d6abeede1ad 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py @@ -13,6 +13,7 @@ import anyio import asyncer import dpath + from pipelines import main_logger from pipelines.models.steps import StepStatus @@ -191,7 +192,6 @@ def _filter_skipped_steps(steps_to_evaluate: STEP_TREE, skip_steps: List[str], r """ steps_to_run: STEP_TREE = [] for step_to_eval in steps_to_evaluate: - # ignore nested steps if isinstance(step_to_eval, list): steps_to_run.append(step_to_eval) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py index 0917f035ea6c..a73cf7f686ec 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py @@ -8,6 +8,7 @@ from google.cloud import storage # type: ignore from google.oauth2 import service_account # type: ignore + from pipelines import main_logger from pipelines.consts import GCS_PUBLIC_DOMAIN diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py index 9ef656747a5e..926a07a76867 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -7,6 +7,7 @@ import git from dagger import Connection, SessionError + from pipelines.consts import CIContext from pipelines.dagger.containers.git import checked_out_git_container from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index ac48aaa8625a..914ecd8c3d64 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -14,6 +14,7 @@ import github as github_sdk from connector_ops.utils import console # type: ignore + from pipelines import main_logger from pipelines.consts import CIContext from pipelines.models.secrets import Secret @@ -107,7 +108,6 @@ def get_pull_request(pull_request_number: int, github_access_token: Secret) -> g def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Optional[Logger] = None) -> None: - update_commit_status_check( click_context["git_revision"], github_state, @@ -138,7 +138,6 @@ def create_or_update_github_pull_request( labels: Optional[Iterable[str]] = None, force_push: bool = True, ) -> github_sdk.PullRequest.PullRequest: - logger = logger or main_logger g = github_sdk.Github(auth=github_sdk.Auth.Token(github_token)) repo = g.get_repo(repo_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py index 28256cc78789..ae190629e364 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, Optional from asyncclick import Command, Context + from pipelines.models.steps import Step diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index 12ae52806787..38c1818b0b7b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -7,6 +7,7 @@ import typing import requests + from pipelines import main_logger if typing.TYPE_CHECKING: diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index ca397153becd..6160924f7a78 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -3,6 +3,7 @@ # """This module groups util function used in pipelines.""" + from __future__ import annotations import contextlib diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py index 4f8776f525fe..295faf9acbcb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py @@ -5,6 +5,7 @@ from typing import Optional import dagger + from pipelines.consts import GCS_PUBLIC_DOMAIN from pipelines.dagger.actions import remote_storage from pipelines.models.secrets import Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index a4c3f623aa53..471b5840addb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -11,9 +11,10 @@ import anyio import dagger from asyncclick import Context, get_current_context +from pydantic import BaseModel, Field, PrivateAttr + from pipelines import main_logger from pipelines.cli.click_decorators import LazyPassDecorator -from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton @@ -86,7 +87,6 @@ async def get_dagger_client(self) -> dagger.Client: if not self._dagger_client: async with self._dagger_client_lock: if not self._dagger_client: - connection = dagger.Connection(dagger.Config(log_output=self.get_log_output())) """ Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py index 323599cc9210..8b816c31334e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -15,10 +15,10 @@ from typing import TYPE_CHECKING, Dict from asyncer import asyncify -from dagger import Client, Directory, File, GitRepository +from dagger import Client, Directory, File, GitRepository, Service from dagger import Secret as DaggerSecret -from dagger import Service from github import PullRequest + from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import MANUAL_PIPELINE_STATUS_CHECK_OVERRIDE_PREFIXES, CIContext, ContextState from pipelines.helpers.execution.run_steps import RunStepOptions @@ -343,7 +343,9 @@ async def __aexit__( if self.should_send_slack_message: # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook await asyncify(send_message_to_webhook)( - self.create_slack_message(), self.get_slack_channels(), self.slack_webhook # type: ignore + self.create_slack_message(), + self.get_slack_channels(), + self.slack_webhook, # type: ignore ) # supress the exception if it was handled return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py index 5d884398653f..408b5ecded6b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py @@ -15,20 +15,22 @@ from typing import List from connector_ops.utils import console # type: ignore -from pipelines.consts import LOCAL_REPORTS_PATH_ROOT -from pipelines.helpers.utils import format_duration, slugify -from pipelines.models.artifacts import Artifact -from pipelines.models.steps import StepResult, StepStatus from rich.console import Group from rich.panel import Panel from rich.style import Style from rich.table import Table from rich.text import Text +from pipelines.consts import LOCAL_REPORTS_PATH_ROOT +from pipelines.helpers.utils import format_duration, slugify +from pipelines.models.artifacts import Artifact +from pipelines.models.steps import StepResult, StepStatus + if typing.TYPE_CHECKING: - from pipelines.models.contexts.pipeline_context import PipelineContext from rich.tree import RenderableType + from pipelines.models.contexts.pipeline_context import PipelineContext + @dataclass(frozen=True) class Report: diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index d46ad47d737d..5f3f5c2bd70b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -16,6 +16,7 @@ import asyncer import click from dagger import Client, Container, DaggerError + from pipelines import main_logger from pipelines.helpers import sentry_utils from pipelines.helpers.utils import format_duration, get_exec_result @@ -23,8 +24,10 @@ from pipelines.models.secrets import Secret if TYPE_CHECKING: + from typing import Any, ClassVar, Optional, Set, Union + import dagger - from typing import Any, ClassVar, Optional, Union, Set + from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.models.contexts.pipeline_context import PipelineContext @@ -120,7 +123,6 @@ def __str__(self) -> str: # noqa D105 @dataclass(kw_only=True, frozen=True) class PoeTaskResult(Result): - task_name: str def __repr__(self) -> str: # noqa D105 @@ -418,7 +420,6 @@ def _get_timed_out_step_result(self) -> StepResult: class StepModifyingFiles(Step): - modified_files: List[str] modified_directory: dagger.Directory diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index fe7314bbebe7..9226b46b7072 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -13,6 +13,7 @@ import pytest import requests from connector_ops.utils import Connector + from pipelines.helpers import utils from tests.utils import ALL_CONNECTORS diff --git a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py index 17f1d746abf3..691de4de1aa5 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py +++ b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py @@ -4,6 +4,7 @@ import pytest from click import UsageError + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python import common from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles @@ -37,8 +38,11 @@ async def test_apply_python_development_overrides( mocker.patch.object(common, "PATH_TO_LOCAL_CDK", local_cdk_path) if local_cdk_is_available: local_cdk_path.mkdir() - await dagger_client.git("https://github.com/airbytehq/airbyte-python-cdk", keep_git_dir=False).branch("main").tree().export( - str(local_cdk_path) + await ( + dagger_client.git("https://github.com/airbytehq/airbyte-python-cdk", keep_git_dir=False) + .branch("main") + .tree() + .export(str(local_cdk_path)) ) connector_context.use_local_cdk = use_local_cdk fake_connector_container = connector_context.dagger_client.container().from_("airbyte/python-connector-base:3.0.0") diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py index 15808f2c88de..5f6045942f35 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_bases.py +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -7,6 +7,7 @@ import anyio import pytest from dagger import DaggerError + from pipelines.models import reports, steps pytestmark = [ diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py index 7b8d28882584..13b2e59d6db4 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, manifest_only_connectors from pipelines.consts import BUILD_PLATFORMS from pipelines.models.steps import StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py index 7b17632d5c48..f03496b7d99a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py @@ -6,6 +6,7 @@ import asyncclick as click import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import BUILD_PLATFORMS diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py index ed0cfe865f35..553dbcea6579 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py @@ -7,6 +7,7 @@ import dagger import docker import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import common from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.models.steps import StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_changelog.py b/airbyte-ci/connectors/pipelines/tests/test_changelog.py index 1ad9bdfe1684..4261048276b1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_changelog.py +++ b/airbyte-ci/connectors/pipelines/tests/test_changelog.py @@ -8,6 +8,7 @@ import pytest import semver + from pipelines.helpers.changelog import Changelog, ChangelogParsingException pytestmark = [ diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index cf503c9d6588..3aa7ecfeecdb 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -5,6 +5,7 @@ import asyncclick as click import pytest from asyncclick.testing import CliRunner + from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index 3b33ce91a425..b3f3b2057712 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -8,6 +8,7 @@ import pytest from asyncclick.testing import CliRunner from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage + from pipelines.airbyte_ci.connectors import commands as connectors_commands from pipelines.airbyte_ci.connectors.build_image import commands as connectors_build_command from pipelines.airbyte_ci.connectors.publish import commands as connectors_publish_command diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index f504b27654ae..2cec51a3ee3f 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -6,6 +6,7 @@ import asyncclick as click import pytest import requests + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python import common @@ -63,7 +64,6 @@ def python_connector_base_image_address(python_connector_with_setup_not_latest_c async def test_with_python_connector_installed_from_setup(context_with_setup, python_connector_base_image_address, latest_cdk_version): - python_container = context_with_setup.dagger_client.container().from_(python_connector_base_image_address) user = await BuildConnectorImages.get_image_user(python_container) container = await common.with_python_connector_installed( diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py index 61ddb9466156..a3c33327516a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py +++ b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py @@ -6,6 +6,7 @@ import pytest from asyncclick.testing import CliRunner + from pipelines.airbyte_ci.format import commands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index 96a397e72855..e0f3604b5448 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -5,8 +5,9 @@ from pathlib import Path -import pipelines.helpers.connectors.modifed import pytest + +import pipelines.helpers.connectors.modifed from pipelines.airbyte_ci.steps import gradle from pipelines.models import steps diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py index 7201a2b83059..a8630cb7988b 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py @@ -5,6 +5,7 @@ import anyio import pytest + from pipelines.helpers.execution import argument_parsing diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py index 30dcc42e9143..069702081d39 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py @@ -4,6 +4,7 @@ import anyio import pytest + from pipelines.helpers.execution.run_steps import InvalidStepConfiguration, RunStepOptions, StepToRun, run_steps from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py index 26605c675849..fe1e88196060 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pytest + from pipelines.helpers.pip import is_package_published diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py index d700a54fe7ad..d13c3ca15493 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py @@ -8,6 +8,7 @@ import dagger import pytest from connector_ops.utils import Connector, ConnectorLanguage + from pipelines import consts from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers import utils @@ -225,7 +226,6 @@ async def test_export_container_to_tarball(mocker, dagger_client, tmp_path, tar_ @pytest.mark.anyio async def test_export_container_to_tarball_failure(mocker, tmp_path): - context = mocker.Mock( connector=mocker.Mock(technical_name="my_connector"), host_image_export_dir_path=tmp_path, diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index dd12b88a930e..3a554f3435ce 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -5,6 +5,7 @@ import asyncclick as click import dagger import pytest + from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py index d3d2f41123ac..65e46ef028fb 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -7,6 +7,7 @@ import pytest import requests from dagger import Client, Platform + from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 57cf9645338e..3c7f1613faa5 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -8,6 +8,7 @@ import anyio import pytest + from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.airbyte_ci.connectors.publish.context import RolloutMode from pipelines.models.steps import StepStatus @@ -361,7 +362,6 @@ async def test_run_connector_python_registry_publish_pipeline( expect_build_connector_called, api_token, ): - for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py index b6b1598a75d9..9671d3e3d648 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest + from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.helpers.utils import get_exec_result from pipelines.models.contexts.pipeline_context import PipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py index 3475fc1d1d9c..02c5276ce478 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py @@ -2,9 +2,10 @@ import pytest from connector_ops.utils import METADATA_FILE_NAME -from pipelines.airbyte_ci.connectors.test.steps.common import VersionIncrementCheck from semver import VersionInfo +from pipelines.airbyte_ci.connectors.test.steps.common import VersionIncrementCheck + class TestVersionIncrementCheck: @pytest.fixture @@ -16,7 +17,6 @@ def context(self, mocker, tmp_path): return context def _get_version_increment_check(self, mocker, context, master_version="1.0.0", current_version="1.0.1"): - mocker.patch( "pipelines.airbyte_ci.connectors.test.steps.common.VersionIncrementCheck.master_connector_version", new_callable=mocker.PropertyMock, diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 90efcfd23f92..4019951179a9 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -12,6 +12,7 @@ import pytest import yaml from freezegun import freeze_time + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps import common from pipelines.dagger.actions.system import docker diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index 5228e34cd28c..7980768e586d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -7,6 +7,7 @@ import asyncclick as click import pytest from connector_ops.utils import Connector, ConnectorLanguage + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps.python_connectors import PyAirbyteValidation, UnitTests diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py index 3beaf70d65a3..4e515605107d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py @@ -13,6 +13,7 @@ import pytest from connector_ops.utils import Connector, ConnectorLanguage from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py index 8f407c30a8c2..9c0047ac4c3a 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py @@ -5,6 +5,7 @@ import inflection import pytest + from connector_acceptance_test.config import Config diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index a29dd584c1a3..6f2a376b73a2 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel + config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration") invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration") spec_path: str = Field( diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 1f29cdaac846..357f38318df9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -17,6 +17,7 @@ import dagger import pytest + from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import ( diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py index 82d32a4aaf52..23665c66165e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py @@ -10,11 +10,13 @@ import pytest from _pytest.config import Config from _pytest.config.argparsing import Parser + from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import Config as AcceptanceTestConfig from connector_acceptance_test.config import GenericTestConfig from connector_acceptance_test.utils import diff_dicts, load_config + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py index df276b655fd7..4b6ed43bfb06 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py @@ -9,9 +9,11 @@ from typing import Any, Dict, List, Mapping import pendulum -from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog from jsonschema import Draft7Validator, FormatChecker, FormatError, ValidationError, validators +from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog + + # fmt: off timestamp_regex = re.compile((r"^\d{4}-\d?\d-\d?\d" # date r"(\s|T)" # separator diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py index e91ee1935415..7b1365f38b18 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py @@ -7,12 +7,13 @@ from typing import Any, Dict import jsonschema -from airbyte_protocol.models import ConnectorSpecification -from connector_acceptance_test.utils import SecretDict from deepdiff import DeepDiff from hypothesis import HealthCheck, Verbosity, given, settings from hypothesis_jsonschema import from_schema +from airbyte_protocol.models import ConnectorSpecification +from connector_acceptance_test.utils import SecretDict + class BackwardIncompatibilityContext(Enum): SPEC = 1 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py index 06276ef4f32d..a6583001a496 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py @@ -10,8 +10,10 @@ from typing import List import dagger + from connector_acceptance_test.utils import SecretDict + IN_CONTAINER_CONNECTOR_PATH = Path("/connector") IN_CONTAINER_CONFIG_PATH = Path("/tmp/config.json") IN_CONTAINER_OUTPUT_PATH = Path("/tmp/output.txt") @@ -55,7 +57,7 @@ async def _run_with_config(container: dagger.Container, command: List[str], conf async def _run(container: dagger.Container, command: List[str]) -> dagger.Container: - return await (container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command)) + return await container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command) async def get_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py index 929309ca0e67..d704b21cf4db 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py @@ -11,6 +11,7 @@ import pytest from yaml import load + try: from yaml import CLoader as Loader except ImportError: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py index 6ae6940f2d1d..74e7a667eac4 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py @@ -11,6 +11,7 @@ import py from pprintpp import pformat + MAX_COLS = py.io.TerminalWriter().fullwidth MARGIN_LEFT = 20 GUTTER = 3 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index 79165b95bf95..6d5ad2994c96 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -14,11 +14,12 @@ import dagger import docker import pytest +from anyio import Path as AnyioPath +from pydantic import ValidationError + from airbyte_protocol.models import AirbyteMessage, ConfiguredAirbyteCatalog, OrchestratorType from airbyte_protocol.models import Type as AirbyteMessageType -from anyio import Path as AnyioPath from connector_acceptance_test.utils import SecretDict -from pydantic import ValidationError def splitlines_generator(input_string: str): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py index 06cd950636fe..c366b6bc7d43 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py @@ -4,6 +4,7 @@ from airbyte_protocol.models import ConnectorSpecification + MANIFEST_FILE_NAMES = [ "manifest.yaml", "manifest.yml", diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py index fa92179b9385..ac1bc6a68bd6 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py @@ -3,6 +3,8 @@ # import pytest +from connector_acceptance_test.utils.asserts import verify_records_schema + from airbyte_protocol.models import ( AirbyteRecordMessage, AirbyteStream, @@ -11,7 +13,6 @@ DestinationSyncMode, SyncMode, ) -from connector_acceptance_test.utils.asserts import verify_records_schema @pytest.fixture(name="record_schema") diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py index 3119a8d43511..ba26286a1cef 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py @@ -6,11 +6,12 @@ from typing import MutableMapping, Union import pytest -from airbyte_protocol.models import AirbyteStream, ConnectorSpecification from connector_acceptance_test.tests.test_core import TestDiscovery as _TestDiscovery from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec from connector_acceptance_test.utils.backward_compatibility import NonBackwardCompatibleError, validate_previous_configs +from airbyte_protocol.models import AirbyteStream, ConnectorSpecification + from .conftest import does_not_raise diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py index 7435915090c7..7fd3d258c88e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -3,10 +3,12 @@ # import pytest -from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type from connector_acceptance_test.config import NoPrimaryKeyConfiguration from connector_acceptance_test.tests import test_core +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type + + pytestmark = pytest.mark.anyio @@ -14,7 +16,9 @@ "stream_configs, excluded_streams, expected_error_streams", [ pytest.param([{"name": "stream_with_primary_key", "primary_key": [["id"]]}], [], None, id="test_stream_with_primary_key_succeeds"), - pytest.param([{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails"), + pytest.param( + [{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails" + ), pytest.param([{"name": "report_stream"}], ["report_stream"], None, id="test_primary_key_excluded_from_test"), pytest.param( [ @@ -22,40 +26,55 @@ {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], [], ["himmel", "heiter"], id="test_multiple_streams_that_are_missing_primary_key"), + ], + [], + ["himmel", "heiter"], + id="test_multiple_streams_that_are_missing_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["himmel", "heiter"], None, id="test_multiple_streams_that_exclude_primary_key"), + ], + ["himmel", "heiter"], + None, + id="test_multiple_streams_that_exclude_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["heiter"], ["himmel"], id="test_multiple_streams_missing_primary_key_or_excluded"), + ], + ["heiter"], + ["himmel"], + id="test_multiple_streams_missing_primary_key_or_excluded", + ), ], ) async def test_streams_define_primary_key(mocker, stream_configs, excluded_streams, expected_error_streams): t = test_core.TestConnectorAttributes() - streams = [AirbyteStream.parse_obj({ - "name": stream_config.get("name"), - "json_schema": {}, - "default_cursor_field": ["updated_at"], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": stream_config.get("primary_key"), - }) for stream_config in stream_configs] + streams = [ + AirbyteStream.parse_obj( + { + "name": stream_config.get("name"), + "json_schema": {}, + "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": stream_config.get("primary_key"), + } + ) + for stream_config in stream_configs + ] streams_without_primary_key = [NoPrimaryKeyConfiguration(name=stream, bypass_reason="") for stream in excluded_streams] docker_runner_mock = mocker.MagicMock( - call_discover=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))] - ) + call_discover=mocker.AsyncMock(return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))]) ) if expected_error_streams: @@ -64,7 +83,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) streams_in_error_message = [stream_name for stream_name in expected_error_streams if stream_name in e.value.args[0]] assert streams_in_error_message == expected_error_streams @@ -73,7 +92,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) @@ -106,26 +125,22 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea "Has `allowdHosts` but no `hosts`", "Has `hosts` but it's empty list", "Has non-empty `hosts`", - ] + ], ) async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) - + @pytest.mark.parametrize( "metadata_yaml, should_raise_assert_error, expected_error", @@ -156,22 +171,18 @@ async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise "Has `suggestedStreams` but no `streams`", "Has `streams` but it's empty list", "Has non-empty `streams`", - ] + ], ) async def test_certified_connector_has_suggested_streams(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml - ) \ No newline at end of file + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml + ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py index c7399c1fbe73..746d6b1166a4 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py @@ -8,6 +8,8 @@ from pathlib import Path import pytest +from connector_acceptance_test.utils import connector_runner + from airbyte_protocol.models import ( AirbyteControlConnectorConfigMessage, AirbyteControlMessage, @@ -16,7 +18,7 @@ OrchestratorType, ) from airbyte_protocol.models import Type as AirbyteMessageType -from connector_acceptance_test.utils import connector_runner + pytestmark = pytest.mark.anyio @@ -121,7 +123,6 @@ def test_persist_new_configuration( async def test_get_connector_container(mocker): - dagger_client = mocker.AsyncMock() os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"] = "test_tarball_path" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index 71799cfead08..d4dcec8c9855 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -6,6 +6,18 @@ from unittest.mock import MagicMock, patch import pytest +from connector_acceptance_test.config import ( + BasicReadTestConfig, + Config, + DiscoveryTestConfig, + ExpectedRecordsConfig, + FileTypesConfig, + IgnoredFieldsConfiguration, + UnsupportedFileTypeConfig, +) +from connector_acceptance_test.tests import test_core +from jsonschema.exceptions import SchemaError + from airbyte_protocol.models import ( AirbyteErrorTraceMessage, AirbyteLogMessage, @@ -28,20 +40,10 @@ TraceType, Type, ) -from connector_acceptance_test.config import ( - BasicReadTestConfig, - Config, - DiscoveryTestConfig, - ExpectedRecordsConfig, - FileTypesConfig, - IgnoredFieldsConfiguration, - UnsupportedFileTypeConfig, -) -from connector_acceptance_test.tests import test_core -from jsonschema.exceptions import SchemaError from .conftest import does_not_raise + pytestmark = pytest.mark.anyio @@ -100,18 +102,11 @@ def test_discovery_uniquely_named_streams(): "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": ["null", "integer"] - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": ["null", "integer"]}}, + }, }, - True + True, ), ( { @@ -119,38 +114,22 @@ def test_discovery_uniquely_named_streams(): "type": ["null", "object"], "properties": { "amount": "integer", - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - True + True, ), ( { "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - False - ) + False, + ), ], ) def test_streams_have_valid_json_schemas(schema, should_fail): @@ -639,9 +618,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -658,9 +635,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -673,9 +648,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -688,9 +661,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -707,9 +678,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -722,9 +691,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -883,7 +850,7 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca _DEFAULT_RECORD_CONFIG, [ {"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}, - {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]} + {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]}, ], {"test_stream": [{"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}]}, None, @@ -911,13 +878,13 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca ), # Expected is in actual but not in order (for case when exact_order=True) ( - {"type": "object"}, - {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, - ExpectedRecordsConfig(exact_order=True, path="foobar"), - [{"constant_field": "not in order"}, {"constant_field": "must equal"}], - {"test_stream": [{"constant_field": "must equal"}]}, - None, - does_not_raise(), + {"type": "object"}, + {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, + ExpectedRecordsConfig(exact_order=True, path="foobar"), + [{"constant_field": "not in order"}, {"constant_field": "must equal"}], + {"test_stream": [{"constant_field": "must equal"}]}, + None, + does_not_raise(), ), # Match by primary key ( @@ -985,12 +952,14 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({ - "name": "test_stream", - "json_schema": schema, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": primary_key - }), + stream=AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": schema, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": primary_key, + } + ), sync_mode="full_refresh", destination_sync_mode="overwrite", ) @@ -998,7 +967,10 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor ) docker_runner_mock = mocker.MagicMock( call_read=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) for record in records] + return_value=[ + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) + for record in records + ] ) ) t = test_core.TestBasicRead() @@ -1954,8 +1926,8 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p ] ) stream = AirbyteStreamState( - stream_descriptor=StreamDescriptor(name='test_stream_0', namespace=None), - stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True) + stream_descriptor=StreamDescriptor(name="test_stream_0", namespace=None), + stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True), ) async_stream_output = [ AirbyteMessage( @@ -1989,7 +1961,7 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.COMPLETE ), ), - ) + ), ] if not state_message_params: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py index 5a602e85a2e7..800860214f62 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py @@ -3,27 +3,28 @@ from pathlib import Path import pytest -from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test import conftest from connector_acceptance_test.tests.test_core import TestConnectorDocumentation as _TestConnectorDocumentation +from airbyte_protocol.models import ConnectorSpecification + @pytest.mark.parametrize( "connector_spec, docs_path, should_fail", ( - # SUCCESS: required field from spec exists in Prerequisites section - ( - {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, - "data/docs/incorrect_not_all_structure.md", - False - ), - # FAIL: required field from spec does not exist in Prerequisites section - ( - {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, - "data/docs/incorrect_not_all_structure.md", - True - ) - ) + # SUCCESS: required field from spec exists in Prerequisites section + ( + {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, + "data/docs/incorrect_not_all_structure.md", + False, + ), + # FAIL: required field from spec does not exist in Prerequisites section + ( + {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + ), ) def test_documentation_prerequisites_section(connector_spec, docs_path, should_fail): t = _TestConnectorDocumentation() @@ -41,35 +42,35 @@ def test_documentation_prerequisites_section(connector_spec, docs_path, should_f @pytest.mark.parametrize( "metadata, docs_path, should_fail, failure", ( - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - "Missing headers:", - ), - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "Oracle Netsuite"}}, - "data/docs/with_not_required_steps.md", - True, - "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", - ), - # # SUCCESS: Docs follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - "", - ), - # Fail: Incorrect header order - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_header_order.md", - True, - "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", - ), - ) + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + "Missing headers:", + ), + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "Oracle Netsuite"}}, + "data/docs/with_not_required_steps.md", + True, + "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", + ), + # # SUCCESS: Docs follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + "", + ), + # Fail: Incorrect header order + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_header_order.md", + True, + "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", + ), + ), ) def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, failure): t = _TestConnectorDocumentation() @@ -89,25 +90,25 @@ def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, fai @pytest.mark.parametrize( "metadata, docs_path, should_fail", ( - # FAIL: Prerequisites section does not follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct_all_description_exist.md", - False, - ), - ) + # FAIL: Prerequisites section does not follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct_all_description_exist.md", + False, + ), + ), ) def test_docs_description(mocker, metadata, docs_path, should_fail): mocker.patch.object(conftest.pytest, "fail") @@ -140,7 +141,7 @@ def test_docs_description(mocker, metadata, docs_path, should_fail): "data/docs/correct.md", False, ), - ) + ), ) def test_docs_urls(docs_path, should_fail): t = _TestConnectorDocumentation() diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py index c6c617be8693..bc76474d1c75 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py @@ -6,7 +6,6 @@ import time import pytest -from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test import conftest from connector_acceptance_test.config import ( BasicReadTestConfig, @@ -16,6 +15,8 @@ IgnoredFieldsConfiguration, ) +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.mark.parametrize( "test_strictness_level, basic_read_test_config, expect_test_failure", diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index 05724361dab0..7b9307f26fc3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -11,6 +11,17 @@ from unittest.mock import MagicMock, patch import pytest +from connector_acceptance_test.config import ( + Config, + EmptyStreamConfiguration, + FutureStateConfig, + FutureStateCursorFormatConfiguration, + IncrementalConfig, +) +from connector_acceptance_test.tests import test_incremental +from connector_acceptance_test.tests.test_incremental import TestIncremental as _TestIncremental +from connector_acceptance_test.tests.test_incremental import future_state_configuration_fixture, future_state_fixture + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,16 +38,7 @@ SyncMode, Type, ) -from connector_acceptance_test.config import ( - Config, - EmptyStreamConfiguration, - FutureStateConfig, - FutureStateCursorFormatConfiguration, - IncrementalConfig, -) -from connector_acceptance_test.tests import test_incremental -from connector_acceptance_test.tests.test_incremental import TestIncremental as _TestIncremental -from connector_acceptance_test.tests.test_incremental import future_state_configuration_fixture, future_state_fixture + pytestmark = [ pytest.mark.anyio, @@ -56,7 +58,10 @@ def build_state_message(state: dict) -> AirbyteMessage: def build_per_stream_state_message( - descriptor: StreamDescriptor, stream_state: Optional[dict[str, Any]], data: Optional[dict[str, Any]] = None, source_stats: Optional[dict[str, Any]] = None + descriptor: StreamDescriptor, + stream_state: Optional[dict[str, Any]], + data: Optional[dict[str, Any]] = None, + source_stats: Optional[dict[str, Any]] = None, ) -> AirbyteMessage: if data is None: data = stream_state @@ -70,7 +75,7 @@ def build_per_stream_state_message( type=AirbyteStateType.STREAM, stream=AirbyteStreamState(stream_descriptor=descriptor, stream_state=stream_state_blob), sourceStats=AirbyteStateStats(**source_stats), - data=data + data=data, ), ) @@ -232,20 +237,40 @@ async def test_incremental_two_sequential_reads( [ {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, ], # Read after 2022-05-10. This is the second (last) subsequent read. [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, - ] + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, + ], ], IncrementalConfig(), does_not_raise(), @@ -258,9 +283,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 0.0}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_records_on_first_read_skips_stream", @@ -274,9 +297,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_states_on_first_read_skips_stream", @@ -293,13 +314,28 @@ async def test_incremental_two_sequential_reads( ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-08"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], ], IncrementalConfig(), @@ -373,7 +409,10 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse call_read_output_messages = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), ) if record["type"] == Type.STATE else build_record_message(record["name"], record["data"]) @@ -382,7 +421,10 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse call_read_with_state_output_messages = [ [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), ) if record["type"] == Type.STATE else build_record_message(stream=record["name"], data=record["data"]) @@ -416,22 +458,20 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse [ {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, ], [], [], - id="combine_three_duplicates_into_a_single_state_message" + id="combine_three_duplicates_into_a_single_state_message", ), pytest.param( [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}} - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, ], + [{"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}], [0.0], - id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal" + id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal", ), pytest.param( [ @@ -443,27 +483,33 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], - [10.0, 3.0, 0.0] - ) + [10.0, 3.0, 0.0], + ), ], ) async def test_get_unique_state_messages(non_unique_states, expected_unique_states, expected_record_count_per_state): non_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in non_unique_states ] expected_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in expected_unique_states ] @@ -471,7 +517,9 @@ async def test_get_unique_state_messages(non_unique_states, expected_unique_stat assert len(actual_unique_states) == len(expected_unique_states) if len(expected_unique_states): - for actual_state_data, expected_state, expected_record_count in zip(actual_unique_states, expected_unique_states, expected_record_count_per_state): + for actual_state_data, expected_state, expected_record_count in zip( + actual_unique_states, expected_unique_states, expected_record_count_per_state + ): actual_state, actual_record_count = actual_state_data assert actual_state == expected_state assert actual_record_count == expected_record_count @@ -517,7 +565,8 @@ async def test_config_skip_test(mocker): IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration())), [], {"type": "object", "properties": {"date": {"type": "str"}}}, - pytest.raises(AssertionError), id="Error because incremental stream should always emit state messages" + pytest.raises(AssertionError), + id="Error because incremental stream should always emit state messages", ), pytest.param( [ @@ -561,20 +610,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -594,25 +632,16 @@ async def test_config_skip_test(mocker): ), ) ], - IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$"))), + IncrementalConfig( + future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$")) + ), [ { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -637,20 +666,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-05-08T03:04:45.139-0700" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-05-08T03:04:45.139-0700"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -676,20 +694,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": 10000000.0 - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": 10000000.0}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": ["int", "null"]}}}, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py index 89536ae24542..8cfcd6d21bb7 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py @@ -7,6 +7,9 @@ import pendulum import pytest +from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure +from pydantic import BaseModel + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,6 @@ SyncMode, Type, ) -from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure -from pydantic import BaseModel def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py index 188b8e39bc18..10659cc83258 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py @@ -5,6 +5,7 @@ import pytest from connector_acceptance_test import config, plugin + HIGH_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.high LOW_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.low diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 4b4b1d456e75..50ea4c0efa25 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -5,10 +5,11 @@ from typing import Any, Callable, Dict import pytest -from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test import conftest from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec +from airbyte_protocol.models import ConnectorSpecification + from .conftest import does_not_raise @@ -681,7 +682,7 @@ def test_enum_usage(connector_spec, should_fail): }, ), "", - ) + ), ], ) def test_validate_oauth_flow(connector_spec, expected_error): @@ -698,227 +699,153 @@ def test_validate_oauth_flow(connector_spec, expected_error): [ # FAIL: OAuth is not default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, }, - ] - } - } + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "Oauth method should be a default option. Current default method is access_token." + }, + }, + }, + ), + "Oauth method should be a default option. Current default method is access_token.", ), # SUCCESS: Oauth is default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - } - ] - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { "type": "string", "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "" + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "", ), # SUCCESS: no advancedAuth specified (ConnectorSpecification(connectionSpecification={}), ""), @@ -992,60 +919,60 @@ def test_validate_oauth_flow(connector_spec, expected_error): ), # SUCCESS: Skipped: no predicate key. ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": {"type": "object"}, - "credentials": { - "type": "object", - "properties": { - "auth_type": {"type": "string", "const": "oauth2.0"}, - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - "token_expiry_date": {"type": "string", "format": "date-time"}, - }, + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, }, }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, - "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"], - }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, - "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, - }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, }, }, - ), - "Advanced Auth object does not have predicate_key, only one option to authenticate.", - ) - ] + }, + ), + "Advanced Auth object does not have predicate_key, only one option to authenticate.", + ), + ], ) def test_validate_auth_default_method(connector_spec, expected_error): t = _TestSpec() @@ -1106,69 +1033,66 @@ def test_additional_properties_is_true(connector_spec, expectation): ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string"}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": {"client_secret": {"type": "string"}, "access_token": {"type": "string", "airbyte_secret": True}}, + } + }, }, - True + True, ), ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string", "airbyte_secret": True}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": { + "client_secret": {"type": "string", "airbyte_secret": True}, + "access_token": {"type": "string", "airbyte_secret": True}, + }, + } + }, }, - False + False, ), ( - { - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - ] - } - } + { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + ], + } }, - True + }, + True, ), ({"type": "object", "properties": {"auth": {"oneOf": [{"api_token": {"type": "string"}}]}}}, True), ( @@ -1187,7 +1111,9 @@ def test_airbyte_secret(mocker, connector_spec, should_fail): t = _TestSpec() logger = mocker.Mock() t.test_secret_is_properly_marked( - {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret") + {"connectionSpecification": connector_spec}, + logger, + ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret"), ) if should_fail: conftest.pytest.fail.assert_called_once() diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py index d767ffa3a0f6..bf8ca2d41c12 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py @@ -7,6 +7,9 @@ import pytest from _pytest.outcomes import Failed +from connector_acceptance_test.config import ConnectionTestConfig, IgnoredFieldsConfiguration +from connector_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,7 @@ SyncMode, Type, ) -from connector_acceptance_test.config import ConnectionTestConfig, IgnoredFieldsConfiguration -from connector_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh + pytestmark = pytest.mark.anyio diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py index 4316d871ca71..d9e1628a29a0 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py @@ -14,11 +14,12 @@ import pytest import yaml -from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test.config import EmptyStreamConfiguration from connector_acceptance_test.utils import common from connector_acceptance_test.utils.compare import make_hashable +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + def not_sorted_data(): return { diff --git a/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py index 9c063d1a2226..1ce3a6008fc0 100644 --- a/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py +++ b/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py index de6cab2f4cbd..43a585a395d4 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py @@ -9,9 +9,10 @@ from uuid import uuid4 import boto3 +from botocore.exceptions import ClientError + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type -from botocore.exceptions import ClientError class DestinationAmazonSqs(Destination): @@ -80,7 +81,6 @@ def set_message_fifo_properties(self, message, message_group_id, use_content_ded def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - # Required propeties queue_url = config["queue_url"] queue_region = config["region"] diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py index 1517a69e1bcc..4af175c0cc5d 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py @@ -7,9 +7,10 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, Status, SyncMode from destination_amazon_sqs import DestinationAmazonSqs +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, Status, SyncMode + @pytest.fixture(name="config") def config_fixture() -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/main.py b/airbyte-integrations/connectors/destination-amazon-sqs/main.py index bc6076972a29..2761b5787225 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/main.py @@ -7,5 +7,6 @@ from destination_amazon_sqs import DestinationAmazonSqs + if __name__ == "__main__": DestinationAmazonSqs().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py index 9ebad9efae7d..e0d84ddc072d 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py @@ -8,13 +8,14 @@ from typing import Any, Mapping import boto3 -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status from destination_amazon_sqs import DestinationAmazonSqs # from airbyte_cdk.sources.source import Source from moto import mock_iam, mock_sqs from moto.core import set_initial_no_auth_action_count +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status + @mock_iam def create_user_with_all_permissions(): diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py index 01d805ecd782..6e36fa02ba5e 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class AstraIndexingModel(BaseModel): astra_db_app_token: str = Field( diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py index 1c79c9aa1d48..20a365c8c880 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py @@ -16,6 +16,7 @@ from destination_astra.config import ConfigModel from destination_astra.indexer import AstraIndexer + BATCH_SIZE = 100 diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py index ee936ae1f5b4..8c64860c8c5f 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py @@ -6,6 +6,7 @@ from typing import Optional import urllib3 + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception @@ -13,6 +14,7 @@ from destination_astra.astra_client import AstraClient from destination_astra.config import AstraIndexingModel + # do not flood the server with too many connections in parallel PARALLELISM_LIMIT = 20 diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py index b9d1aac8ae3c..2ab044fbdf47 100644 --- a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -4,27 +4,26 @@ import logging -from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_astra.astra_client import AstraClient from destination_astra.config import ConfigModel from destination_astra.destination import DestinationAstra +from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status -class AstraIntegrationTest(BaseIntegrationTest): +class AstraIntegrationTest(BaseIntegrationTest): def test_check_valid_config(self): outcome = DestinationAstra().check(logging.getLogger("airbyte"), self.config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): - invalid_config = self.config + invalid_config = self.config invalid_config["embedding"]["openai_key"] = 123 - outcome = DestinationAstra().check( - logging.getLogger("airbyte"), invalid_config) + outcome = DestinationAstra().check(logging.getLogger("airbyte"), invalid_config) assert outcome.status == Status.FAILED def test_write(self): @@ -32,11 +31,7 @@ def test_write(self): embedder = create_from_config(db_config.embedding, db_config.processing) db_creds = db_config.indexing astra_client = AstraClient( - db_creds.astra_db_endpoint, - db_creds.astra_db_app_token, - db_creds.astra_db_keyspace, - embedder.embedding_dimensions, - "cosine" + db_creds.astra_db_endpoint, db_creds.astra_db_app_token, db_creds.astra_db_keyspace, embedder.embedding_dimensions, "cosine" ) astra_client.delete_documents(collection_name=db_creds.collection, filter={}) @@ -49,4 +44,3 @@ def test_write(self): outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) assert astra_client.count_documents(db_creds.collection) == 2 - diff --git a/airbyte-integrations/connectors/destination-astra/main.py b/airbyte-integrations/connectors/destination-astra/main.py index 53b96b2b39ec..f56f6b8ac36a 100644 --- a/airbyte-integrations/connectors/destination-astra/main.py +++ b/airbyte-integrations/connectors/destination-astra/main.py @@ -7,5 +7,6 @@ from destination_astra import DestinationAstra + if __name__ == "__main__": DestinationAstra().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py index a4094d41f695..b25fe4966b32 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_astra.config import ConfigModel from destination_astra.destination import DestinationAstra +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationAstra(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py index ebb1d41e230a..312956df5006 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -5,10 +5,11 @@ import pytest import urllib3 -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_astra.config import AstraIndexingModel from destination_astra.indexer import AstraIndexer +from airbyte_cdk.models import ConfiguredAirbyteCatalog + def create_astra_indexer(): config = AstraIndexingModel( @@ -142,7 +143,9 @@ def test_astra_pre_sync(): ("other_collection", None, 3, False, "mycollection collection does not exist."), ( ["mycollection"], - urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.")), + urllib3.exceptions.MaxRetryError( + None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.") + ), 3, False, "Failed to resolve environment", @@ -157,12 +160,16 @@ def test_astra_check(collection_name, describe_throws, reported_dimensions, chec indexer.client.create_collection = MagicMock() indexer.client.find_collections = MagicMock() - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] if describe_throws: indexer.client.find_collections.side_effect = describe_throws else: - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] result = indexer.check() if check_succeeds: diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py index 0c72637d5c84..22de92c7373c 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py @@ -10,15 +10,17 @@ import boto3 import botocore import pandas as pd -from airbyte_cdk.destinations import Destination from awswrangler import _data_types from botocore.credentials import AssumeRoleCredentialFetcher, CredentialResolver, DeferredRefreshableCredentials, JSONFileCache from botocore.exceptions import ClientError from retrying import retry +from airbyte_cdk.destinations import Destination + from .config_reader import CompressionCodec, ConnectorConfig, CredentialsType, OutputFormat from .constants import BOOLEAN_VALUES, EMPTY_VALUES + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py index 223312f5d05b..72cd4acebb2d 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py @@ -8,14 +8,16 @@ from typing import Any, Dict, Iterable, Mapping import pandas as pd +from botocore.exceptions import ClientError, InvalidRegionError + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, AirbyteStateType, ConfiguredAirbyteCatalog, Status, Type -from botocore.exceptions import ClientError, InvalidRegionError from .aws import AwsHandler from .config_reader import ConnectorConfig from .stream_writer import StreamWriter + logger = logging.getLogger("airbyte") # Flush records every 25000 records to limit memory consumption @@ -34,7 +36,6 @@ def _get_random_string(length: int) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py index d7ee96e27688..ba1972b24a83 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py @@ -9,12 +9,14 @@ from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd + from airbyte_cdk.models import ConfiguredAirbyteStream, DestinationSyncMode from .aws import AwsHandler from .config_reader import ConnectorConfig, PartitionOptions from .constants import EMPTY_VALUES, GLUE_TYPE_MAPPING_DECIMAL, GLUE_TYPE_MAPPING_DOUBLE, PANDAS_TYPE_MAPPING + # By default we set glue decimal type to decimal(28,25) # this setting matches that precision. getcontext().prec = 25 diff --git a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py index 29f792fbc02e..4b871b2dc667 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py @@ -9,6 +9,10 @@ import awswrangler as wr import pytest +from destination_aws_datalake import DestinationAwsDatalake +from destination_aws_datalake.aws import AwsHandler +from destination_aws_datalake.config_reader import ConnectorConfig + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -24,9 +28,7 @@ SyncMode, Type, ) -from destination_aws_datalake import DestinationAwsDatalake -from destination_aws_datalake.aws import AwsHandler -from destination_aws_datalake.config_reader import ConnectorConfig + logger = logging.getLogger("airbyte") @@ -95,13 +97,13 @@ def test_check_invalid_aws_account_config(invalid_account_config: Mapping): def _state(stream: str, data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage( - type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_state=data, - stream_descriptor=StreamDescriptor(name=stream, namespace=None) - ) - )) + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState(stream_state=data, stream_descriptor=StreamDescriptor(name=stream, namespace=None)), + ), + ) def _record(stream: str, str_value: str, int_value: int, date_value: datetime) -> AirbyteMessage: diff --git a/airbyte-integrations/connectors/destination-aws-datalake/main.py b/airbyte-integrations/connectors/destination-aws-datalake/main.py index 23a5962dcc72..0aca839fd214 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/main.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/main.py @@ -7,5 +7,6 @@ from destination_aws_datalake import DestinationAwsDatalake + if __name__ == "__main__": DestinationAwsDatalake().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py index e981907cec40..d27168e067ca 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py @@ -9,12 +9,13 @@ import numpy as np import pandas as pd -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from destination_aws_datalake import DestinationAwsDatalake from destination_aws_datalake.aws import AwsHandler from destination_aws_datalake.config_reader import ConnectorConfig from destination_aws_datalake.stream_writer import DictEncoder, StreamWriter +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + def get_config() -> Mapping[str, Any]: with open("unit_tests/fixtures/config.json", "r") as f: diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py index 557d4f527986..3d41cfd6a6a1 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py @@ -4,6 +4,8 @@ from typing import Literal, Optional, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import ( AzureOpenAIEmbeddingConfigModel, CohereEmbeddingConfigModel, @@ -13,7 +15,6 @@ OpenAIEmbeddingConfigModel, VectorDBConfigModel, ) -from pydantic import BaseModel, Field class HttpMode(BaseModel): @@ -41,7 +42,6 @@ class Config: class ChromaIndexingConfigModel(BaseModel): - auth_method: Union[PersistentMode, HttpMode] = Field( ..., title="Connection Mode", description="Mode how to connect to Chroma", discriminator="mode", type="object", order=0 ) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py index b0926ffd3c65..7d04bfaa045d 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -23,11 +23,11 @@ from destination_chroma.indexer import ChromaIndexer from destination_chroma.no_embedder import NoEmbedder + BATCH_SIZE = 128 class DestinationChroma(Destination): - indexer: Indexer embedder: Embedder @@ -42,7 +42,6 @@ def _init_indexer(self, config: ConfigModel): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py index 3b0d741d7feb..10ed56ec44b9 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py @@ -6,12 +6,13 @@ import uuid import chromadb +from chromadb.config import Settings + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode -from chromadb.config import Settings from destination_chroma.config import ChromaIndexingConfigModel from destination_chroma.utils import is_valid_collection_name diff --git a/airbyte-integrations/connectors/destination-chroma/main.py b/airbyte-integrations/connectors/destination-chroma/main.py index 88af98a9500c..146cc0699350 100644 --- a/airbyte-integrations/connectors/destination-chroma/main.py +++ b/airbyte-integrations/connectors/destination-chroma/main.py @@ -7,5 +7,6 @@ from destination_chroma import DestinationChroma + if __name__ == "__main__": DestinationChroma().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py index 52460e533172..b7c3256fc450 100644 --- a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_chroma.config import ConfigModel from destination_chroma.destination import DestinationChroma +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationChroma(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py index f1a9bf493d57..88a1160496dc 100644 --- a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py @@ -5,10 +5,11 @@ import unittest from unittest.mock import Mock -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode from destination_chroma.config import ChromaIndexingConfigModel from destination_chroma.indexer import ChromaIndexer +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode + class TestChromaIndexer(unittest.TestCase): def setUp(self): @@ -30,7 +31,6 @@ def setUp(self): self.mock_client.get_collection = Mock() def test_valid_collection_name(self): - test_configs = [ ({"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, None), ( diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/client.py b/airbyte-integrations/connectors/destination-convex/destination_convex/client.py index 3cace64444da..00aba7fbcff2 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/client.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/client.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping import requests + from destination_convex.config import ConvexConfig diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/config.py b/airbyte-integrations/connectors/destination-convex/destination_convex/config.py index 1c5dc8ca2bcc..3eba952a5e36 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/config.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/config.py @@ -4,6 +4,7 @@ from typing import TypedDict + ConvexConfig = TypedDict( "ConvexConfig", { diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py b/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py index 4a139ceae895..284916c8fb37 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, Optional, cast import requests + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import ( AirbyteConnectionStatus, diff --git a/airbyte-integrations/connectors/destination-convex/main.py b/airbyte-integrations/connectors/destination-convex/main.py index 7a572b8ee791..2426c8f42fa3 100644 --- a/airbyte-integrations/connectors/destination-convex/main.py +++ b/airbyte-integrations/connectors/destination-convex/main.py @@ -7,5 +7,6 @@ from destination_convex import DestinationConvex + if __name__ == "__main__": DestinationConvex().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py index d39027b38692..534dafa72859 100644 --- a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py @@ -7,6 +7,10 @@ import pytest import responses +from destination_convex.client import ConvexClient +from destination_convex.config import ConvexConfig +from destination_convex.destination import DestinationConvex + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -18,9 +22,7 @@ SyncMode, Type, ) -from destination_convex.client import ConvexClient -from destination_convex.config import ConvexConfig -from destination_convex.destination import DestinationConvex + DEDUP_TABLE_NAME = "dedup_stream" DEDUP_INDEX_FIELD = "int_col" diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py index 10728e374f54..faa12d0c8957 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py @@ -9,6 +9,7 @@ from cumulio.cumulio import Cumulio # type: ignore + # def _retry_with_backoff( # fn: Callable, # backoff_times_in_seconds: list[int] diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py index 61c6c5ac4afb..68ea73a6fdf0 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py @@ -11,6 +11,7 @@ from destination_cumulio.client import CumulioClient from destination_cumulio.writer import CumulioWriter + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py index 545241d463e7..1ecf57279111 100644 --- a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py @@ -8,6 +8,9 @@ from typing import Any, Dict, Mapping import pytest +from destination_cumulio import DestinationCumulio +from destination_cumulio.client import CumulioClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -20,8 +23,6 @@ SyncMode, Type, ) -from destination_cumulio import DestinationCumulio -from destination_cumulio.client import CumulioClient @pytest.fixture(name="logger") diff --git a/airbyte-integrations/connectors/destination-cumulio/main.py b/airbyte-integrations/connectors/destination-cumulio/main.py index 3ad0d7112206..ef230369f793 100644 --- a/airbyte-integrations/connectors/destination-cumulio/main.py +++ b/airbyte-integrations/connectors/destination-cumulio/main.py @@ -7,5 +7,6 @@ from destination_cumulio import DestinationCumulio + if __name__ == "__main__": DestinationCumulio().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py index 258e8ff2a578..48f538b20d67 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py @@ -9,6 +9,7 @@ import pytest from destination_cumulio.client import CumulioClient + # "# type: ignore" was added in several places to avoid mypy complaining about patching functions with MagicMock diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py index 4805fb51ecf5..2c278430dd6c 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py @@ -9,6 +9,8 @@ from unittest.mock import MagicMock, call, patch import pytest +from destination_cumulio.destination import DestinationCumulio + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -20,7 +22,6 @@ SyncMode, Type, ) -from destination_cumulio.destination import DestinationCumulio @pytest.fixture(name="logger") diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py index ac921c7ef5c4..4c95b3e670e0 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py @@ -8,9 +8,10 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from destination_cumulio.writer import CumulioWriter +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.fixture(name="logger") def logger_fixture() -> MagicMock: diff --git a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py index f8da4b0984bd..01a345b41249 100644 --- a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py +++ b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py @@ -16,6 +16,7 @@ from .writer import create_databend_wirter + logger = getLogger("airbyte") @@ -23,7 +24,6 @@ class DestinationDatabend(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ TODO Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py index a40494c4e048..9bca56ba4224 100644 --- a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from typing import Any, Dict, List, Mapping import pytest +from destination_databend import DestinationDatabend +from destination_databend.client import DatabendClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_databend import DestinationDatabend -from destination_databend.client import DatabendClient @pytest.fixture(name="databendConfig") diff --git a/airbyte-integrations/connectors/destination-databend/main.py b/airbyte-integrations/connectors/destination-databend/main.py index 7482c00577de..67d4d20d9cad 100644 --- a/airbyte-integrations/connectors/destination-databend/main.py +++ b/airbyte-integrations/connectors/destination-databend/main.py @@ -7,5 +7,6 @@ from destination_databend import DestinationDatabend + if __name__ == "__main__": DestinationDatabend().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py index 2372f49736fb..8047bef61ebf 100644 --- a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py +++ b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py @@ -6,6 +6,9 @@ from typing import Dict from unittest.mock import AsyncMock, MagicMock, call, patch +from destination_databend.destination import DatabendClient, DestinationDatabend +from pytest import fixture + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,6 @@ SyncMode, Type, ) -from destination_databend.destination import DatabendClient, DestinationDatabend -from pytest import fixture @fixture @@ -117,14 +118,14 @@ def test_connection(config: Dict[str, str], logger: MagicMock) -> None: @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_append( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ) -> None: catalog = ConfiguredAirbyteCatalog(streams=[configured_stream1, configured_stream2]) @@ -141,14 +142,14 @@ def test_sql_write_append( @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_overwrite( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ): # Overwrite triggers a delete configured_stream1.destination_sync_mode = DestinationSyncMode.overwrite diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index 8e74038d41e6..facbeb3344c9 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -2,6 +2,7 @@ import datetime import json +import logging import os import re import uuid @@ -12,7 +13,6 @@ import duckdb import pyarrow as pa -import logging from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py index d985da9706db..b3320efb79fe 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py @@ -15,6 +15,10 @@ import duckdb import pytest +from destination_duckdb import DestinationDuckdb +from destination_duckdb.destination import CONFIG_MOTHERDUCK_API_KEY +from faker import Faker + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,14 +31,10 @@ SyncMode, Type, ) -from destination_duckdb import DestinationDuckdb -from destination_duckdb.destination import CONFIG_MOTHERDUCK_API_KEY -from faker import Faker + CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +45,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -102,6 +100,7 @@ def test_large_table_name() -> str: rand_string = "".join(random.choice(letters) for _ in range(10)) return f"airbyte_integration_{rand_string}" + @pytest.fixture def table_schema() -> str: schema = {"type": "object", "properties": {"column1": {"type": ["null", "string"]}}} @@ -110,7 +109,9 @@ def table_schema() -> str: @pytest.fixture def configured_catalogue( - test_table_name: str, test_large_table_name: str, table_schema: str, + test_table_name: str, + test_large_table_name: str, + table_schema: str, ) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( stream=AirbyteStream( @@ -159,9 +160,7 @@ def airbyte_message2(test_table_name: str): @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.mark.disable_autouse @@ -208,9 +207,7 @@ def test_write( if motherduck_api_key: duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: cursor = con.execute( "SELECT _airbyte_ab_id, _airbyte_emitted_at, _airbyte_data " @@ -222,21 +219,20 @@ def test_write( assert result[0][2] == json.dumps(airbyte_message1.record.data) assert result[1][2] == json.dumps(airbyte_message2.record.data) + def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, - data={"key1": fake.first_name() , "key2": fake.ssn()}, + data={"key1": fake.first_name(), "key2": fake.ssn()}, emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -250,28 +246,28 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, # Throw in empty nested objects and see how pyarrow deals with them. - data={"key1": fake.first_name(), - "key2": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "nested1": {} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "dictionary1":{} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": "True" if random.random() < 0.5 else True - } - } - } - if random.random() < 0.9 else {}, - + data={ + "key1": fake.first_name(), + "key2": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "nested1": {} + if random.random() < 0.1 + else { + "key3": fake.first_name(), + "key4": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "dictionary1": {} + if random.random() < 0.1 + else {"key3": fake.first_name(), "key4": "True" if random.random() < 0.5 else True}, + }, + } + if random.random() < 0.9 + else {}, emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -281,10 +277,15 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab TOTAL_RECORDS = 5_000 BATCH_WRITE_SIZE = 1000 + @pytest.mark.slow -@pytest.mark.parametrize("airbyte_message_generator,explanation", - [(_airbyte_messages, "Test writing a large number of simple json objects."), - (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema.")] ) +@pytest.mark.parametrize( + "airbyte_message_generator,explanation", + [ + (_airbyte_messages, "Test writing a large number of simple json objects."), + (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema."), + ], +) def test_large_number_of_writes( config: Dict[str, str], request, @@ -309,13 +310,8 @@ def test_large_number_of_writes( duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: - cursor = con.execute( - "SELECT count(1) " - f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}" - ) + cursor = con.execute("SELECT count(1) " f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}") result = cursor.fetchall() assert result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-duckdb/main.py b/airbyte-integrations/connectors/destination-duckdb/main.py index 5aca3c166745..1353a0f6298f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/main.py +++ b/airbyte-integrations/connectors/destination-duckdb/main.py @@ -5,5 +5,6 @@ from destination_duckdb.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py index 4f5aeefa09b7..7a2616579d57 100644 --- a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py @@ -4,6 +4,7 @@ import os import tempfile from unittest.mock import Mock, patch + import pytest from destination_duckdb.destination import CONFIG_DEFAULT_SCHEMA, DestinationDuckdb, validated_sql_name @@ -15,6 +16,7 @@ def test_validated_sql_name() -> None: with pytest.raises(ValueError): validated_sql_name("invalid-name") + @patch("duckdb.connect") @patch("os.makedirs") def test_check(mock_connect, mock_makedirs) -> None: @@ -27,6 +29,7 @@ def test_check(mock_connect, mock_makedirs) -> None: result = destination.check(logger, config) assert result.status == Status.SUCCEEDED + @patch("duckdb.connect") @patch("os.makedirs") def test_check_failure(mock_connect, mock_makedirs) -> None: @@ -38,6 +41,7 @@ def test_check_failure(mock_connect, mock_makedirs) -> None: assert result.status == Status.FAILED assert "Test exception" in result.message + @patch("duckdb.connect") @patch("os.makedirs") def test_write(mock_connect, mock_makedirs) -> None: diff --git a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py index 46d960893fc8..fb040c2ac3cb 100644 --- a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py @@ -9,14 +9,16 @@ from typing import Any, Dict, Iterable, Mapping, Optional from uuid import uuid4 -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from firebolt.client import DEFAULT_API_URL from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword from firebolt.db import Connection, connect +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type + from .writer import create_firebolt_wirter + logger = getLogger("airbyte") @@ -75,7 +77,6 @@ class DestinationFirebolt(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py index 872db32c3821..397298599d37 100644 --- a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py @@ -9,6 +9,10 @@ from typing import Dict from unittest.mock import MagicMock +from destination_firebolt.destination import DestinationFirebolt, establish_connection +from firebolt.common.exception import FireboltError +from pytest import fixture, mark, raises + from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteStream, @@ -17,9 +21,6 @@ DestinationSyncMode, SyncMode, ) -from destination_firebolt.destination import DestinationFirebolt, establish_connection -from firebolt.common.exception import FireboltError -from pytest import fixture, mark, raises @fixture(scope="module") diff --git a/airbyte-integrations/connectors/destination-firebolt/main.py b/airbyte-integrations/connectors/destination-firebolt/main.py index 1b173be0c2b3..99c40026fe63 100644 --- a/airbyte-integrations/connectors/destination-firebolt/main.py +++ b/airbyte-integrations/connectors/destination-firebolt/main.py @@ -7,5 +7,6 @@ from destination_firebolt import DestinationFirebolt + if __name__ == "__main__": DestinationFirebolt().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py index d4252f97d5c1..ec74bd77d285 100644 --- a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py @@ -6,6 +6,9 @@ from typing import Any, Dict from unittest.mock import MagicMock, call, patch +from destination_firebolt.destination import DestinationFirebolt, establish_connection, parse_config +from pytest import fixture + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,8 +20,6 @@ SyncMode, Type, ) -from destination_firebolt.destination import DestinationFirebolt, establish_connection, parse_config -from pytest import fixture @fixture(params=["my_engine", "my_engine.api.firebolt.io"]) @@ -34,6 +35,7 @@ def config(request: Any) -> Dict[str, str]: } return args + @fixture() def legacy_config(): args = { @@ -45,6 +47,7 @@ def legacy_config(): } return args + @fixture def config_external_table() -> Dict[str, str]: args = { diff --git a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py index a063b1efd16c..7fdd8626e712 100644 --- a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py @@ -7,6 +7,10 @@ from typing import Any, Dict, Mapping import pytest +from destination_firestore import DestinationFirestore +from destination_firestore.writer import FirestoreWriter +from google.cloud import firestore + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,9 +23,6 @@ SyncMode, Type, ) -from destination_firestore import DestinationFirestore -from destination_firestore.writer import FirestoreWriter -from google.cloud import firestore @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-firestore/main.py b/airbyte-integrations/connectors/destination-firestore/main.py index e5bf045a303d..6f165c46b28d 100644 --- a/airbyte-integrations/connectors/destination-firestore/main.py +++ b/airbyte-integrations/connectors/destination-firestore/main.py @@ -7,5 +7,6 @@ from destination_firestore import DestinationFirestore + if __name__ == "__main__": DestinationFirestore().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py index 855cba00d92e..e45411c806be 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py @@ -10,7 +10,6 @@ class WriteBufferMixin: - # Default instance of AirbyteLogger logger = AirbyteLogger() # intervals after which the records_buffer should be cleaned up for selected stream diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py index 41c857f03e3e..4d0502b81e0d 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py @@ -6,11 +6,13 @@ from typing import Dict import pygsheets -from airbyte_cdk import AirbyteLogger from google.auth.transport.requests import Request from google.oauth2 import credentials as client_account from pygsheets.client import Client as pygsheets_client +from airbyte_cdk import AirbyteLogger + + # the list of required scopes/permissions # more info: https://developers.google.com/sheets/api/guides/authorizing#OAuth2Authorizing SCOPES = [ @@ -20,7 +22,6 @@ class GoogleSheetsClient: - logger = AirbyteLogger() def __init__(self, config: Dict): diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py index 2e1e4343ec55..f25015f0af75 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py @@ -4,10 +4,11 @@ from typing import Any, Iterable, Mapping +from google.auth.exceptions import RefreshError + from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type -from google.auth.exceptions import RefreshError from .client import GoogleSheetsClient from .helpers import ConnectionTest, get_spreadsheet_id, get_streams_from_catalog @@ -40,7 +41,6 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. """ diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py index 29d27ae744fc..6cd7390e65a6 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py @@ -6,11 +6,13 @@ import re from typing import List -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ConfiguredAirbyteCatalog from pygsheets import Spreadsheet, Worksheet from pygsheets.exceptions import WorksheetNotFound +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + STREAMS_COUNT_LIMIT = 200 @@ -39,7 +41,6 @@ def get_streams_from_catalog(catalog: ConfiguredAirbyteCatalog, limit: int = STR class ConnectionTest: - """ Performs connection test write operation to ensure the target spreadsheet is available for writing. Initiating the class itself, performs the connection test and stores the result in ConnectionTest.result property. diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py index 2d2815bf3244..b6a9ef3814fc 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py @@ -3,9 +3,10 @@ # -from airbyte_cdk.models import AirbyteStream from pygsheets import Worksheet +from airbyte_cdk.models import AirbyteStream + from .buffer import WriteBufferMixin from .spreadsheet import GoogleSheets diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py index ce56ac806ac7..200cea5a12d1 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py @@ -7,9 +7,11 @@ from typing import Iterable import pytest +from destination_google_sheets.buffer import WriteBufferMixin + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type -from destination_google_sheets.buffer import WriteBufferMixin + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py index a79195eba97d..caf0a695329e 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py @@ -8,6 +8,7 @@ from integration_tests.test_helpers import TEST_CONFIG from pygsheets.client import Client as pygsheets_client + # ----- PREPARE ENV ----- # path to configured_catalog json file diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py index d26a46e197fa..e09802ea0057 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py @@ -7,13 +7,15 @@ from io import StringIO import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import AirbyteConnectionStatus, Status from destination_google_sheets.destination import DestinationGoogleSheets from integration_tests.test_buffer import read_input_messages from integration_tests.test_helpers import TEST_CONFIG from integration_tests.test_writer import TEST_CATALOG, TEST_SPREADSHEET, TEST_STREAM +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import AirbyteConnectionStatus, Status + + # ----- PREPARE ENV ----- @@ -63,7 +65,6 @@ def test_check(): ], ) def test_write(expected, raised): - # clean worksheet after previous test TEST_SPREADSHEET.clean_worksheet(TEST_STREAM) diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py index 3495ff513526..af98e6ff12d6 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py @@ -6,12 +6,14 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_google_sheets.client import GoogleSheetsClient from destination_google_sheets.helpers import ConnectionTest, get_spreadsheet_id, get_streams_from_catalog from destination_google_sheets.spreadsheet import GoogleSheets from pygsheets.client import Client as pygsheets_client +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py index 02e9593e6913..2e60c881104d 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py @@ -9,6 +9,7 @@ from integration_tests.test_helpers import TEST_CONFIG from pygsheets.client import Client as pygsheets_client + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py index e4adbe29eb53..c360221840e9 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py @@ -3,10 +3,12 @@ # import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_google_sheets.writer import GoogleSheetsWriter from integration_tests.test_spreadsheet import TEST_SPREADSHEET +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/main.py b/airbyte-integrations/connectors/destination-google-sheets/main.py index c2480d5a1f64..0a94f725540f 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/main.py +++ b/airbyte-integrations/connectors/destination-google-sheets/main.py @@ -7,5 +7,6 @@ from destination_google_sheets import DestinationGoogleSheets + if __name__ == "__main__": DestinationGoogleSheets().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py index cbbb5c90c0df..0a23edbb4b62 100644 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py +++ b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py @@ -19,7 +19,6 @@ class DestinationKvdb(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-kvdb/main.py b/airbyte-integrations/connectors/destination-kvdb/main.py index 178789589e5a..09a8a9219af9 100644 --- a/airbyte-integrations/connectors/destination-kvdb/main.py +++ b/airbyte-integrations/connectors/destination-kvdb/main.py @@ -7,5 +7,6 @@ from destination_kvdb import DestinationKvdb + if __name__ == "__main__": DestinationKvdb().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py index 39aa307918a5..a34b2d738894 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py @@ -4,9 +4,10 @@ import sys -from airbyte_cdk.entrypoint import launch from destination_meilisearch import DestinationMeilisearch +from airbyte_cdk.entrypoint import launch + def run(): source = DestinationMeilisearch() diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py index c1126786b3b6..06fc951b230f 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py @@ -4,9 +4,10 @@ from typing import Literal, Optional, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic import BaseModel, Field class UsernamePasswordAuth(BaseModel): diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py index 6f80dc6706c2..8b390574574c 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py @@ -16,6 +16,7 @@ from destination_milvus.config import ConfigModel from destination_milvus.indexer import MilvusIndexer + BATCH_SIZE = 128 diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py index bca9df695d8c..c88294815f4c 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py @@ -7,13 +7,15 @@ from multiprocessing import Process from typing import Optional +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from destination_milvus.config import MilvusIndexingConfigModel -from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + CLOUD_DEPLOYMENT_MODE = "cloud" diff --git a/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py index 731ba7edbe76..1605cde5b0b0 100644 --- a/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py +++ b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py @@ -5,14 +5,15 @@ import json import logging -from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_milvus.destination import DestinationMilvus from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Milvus from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status + class MilvusIntegrationTest(BaseIntegrationTest): """ diff --git a/airbyte-integrations/connectors/destination-milvus/main.py b/airbyte-integrations/connectors/destination-milvus/main.py index ed23821ba6c3..5ed2b68506e3 100644 --- a/airbyte-integrations/connectors/destination-milvus/main.py +++ b/airbyte-integrations/connectors/destination-milvus/main.py @@ -7,5 +7,6 @@ from destination_milvus import DestinationMilvus + if __name__ == "__main__": DestinationMilvus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py index 982c42d1fa86..4ce5784f760e 100644 --- a/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_milvus.config import ConfigModel from destination_milvus.destination import DestinationMilvus +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationMilvus(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py index f88d064862c3..3087226daf94 100644 --- a/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py @@ -6,11 +6,12 @@ import unittest from unittest.mock import Mock, call, patch -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode from destination_milvus.config import MilvusIndexingConfigModel, NoAuth, TokenAuth from destination_milvus.indexer import MilvusIndexer from pymilvus import DataType +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode + @patch("destination_milvus.indexer.connections") @patch("destination_milvus.indexer.utility") diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py index 879c7a05c2c9..08bdbfa8e56a 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py @@ -15,6 +15,9 @@ from urllib.parse import urlparse import orjson +from serpyco_rs import Serializer +from typing_extensions import override + from airbyte_cdk import AirbyteStream, ConfiguredAirbyteStream, SyncMode from airbyte_cdk.destinations import Destination from airbyte_cdk.exception_handler import init_uncaught_exception_handler @@ -35,8 +38,7 @@ from airbyte_cdk.sql.types import SQLTypeConverter from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor from destination_motherduck.processors.motherduck import MotherDuckConfig, MotherDuckSqlProcessor -from serpyco_rs import Serializer -from typing_extensions import override + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py index a37cb35ebbcf..c25bc81cbdfa 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py @@ -10,17 +10,19 @@ from urllib.parse import parse_qsl, urlparse import pyarrow as pa -from airbyte_cdk import DestinationSyncMode -from airbyte_cdk.sql import exceptions as exc -from airbyte_cdk.sql.constants import AB_EXTRACTED_AT_COLUMN, DEBUG_MODE -from airbyte_cdk.sql.secrets import SecretString -from airbyte_cdk.sql.shared.sql_processor import SqlConfig, SqlProcessorBase, SQLRuntimeError from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from sqlalchemy import Executable, TextClause, create_engine, text from sqlalchemy.exc import ProgrammingError, SQLAlchemyError +from airbyte_cdk import DestinationSyncMode +from airbyte_cdk.sql import exceptions as exc +from airbyte_cdk.sql.constants import AB_EXTRACTED_AT_COLUMN, DEBUG_MODE +from airbyte_cdk.sql.secrets import SecretString +from airbyte_cdk.sql.shared.sql_processor import SqlConfig, SqlProcessorBase, SQLRuntimeError + + if TYPE_CHECKING: from sqlalchemy.engine import Connection, Engine diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py index d05a75a56a7c..6da7f8417674 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py @@ -18,14 +18,16 @@ import warnings -from airbyte_cdk.sql.constants import DEBUG_MODE -from airbyte_cdk.sql.secrets import SecretString -from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from sqlalchemy import Engine, create_engine +from airbyte_cdk.sql.constants import DEBUG_MODE +from airbyte_cdk.sql.secrets import SecretString +from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor + + # Suppress warnings from DuckDB about reflection on indices. # https://github.com/Mause/duckdb_engine/issues/905 warnings.filterwarnings( diff --git a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py index 076bc7014685..9ae7e741e9bd 100644 --- a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py @@ -14,6 +14,10 @@ import duckdb import pytest +from destination_motherduck import DestinationMotherDuck +from destination_motherduck.destination import CONFIG_MOTHERDUCK_API_KEY +from faker import Faker + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,14 +31,10 @@ Type, ) from airbyte_cdk.sql.secrets import SecretString -from destination_motherduck import DestinationMotherDuck -from destination_motherduck.destination import CONFIG_MOTHERDUCK_API_KEY -from faker import Faker + CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +45,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -89,9 +87,7 @@ def disable_destination_modification(monkeypatch, request): if "disable_autouse" in request.keywords: return else: - monkeypatch.setattr( - DestinationMotherDuck, "_get_destination_path", lambda _, x: x - ) + monkeypatch.setattr(DestinationMotherDuck, "_get_destination_path", lambda _, x: x) @pytest.fixture(scope="module") @@ -254,9 +250,7 @@ def airbyte_message2_update(airbyte_message2: AirbyteMessage, test_table_name: s @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.fixture @@ -308,11 +302,7 @@ def _state(data: Dict[str, Any]) -> AirbyteMessage: @pytest.fixture() -def sql_processor( - configured_catalogue, - test_schema_name, - config: Dict[str, str] - ): +def sql_processor(configured_catalogue, test_schema_name, config: Dict[str, str]): destination = DestinationMotherDuck() path = config.get("destination_path", "md:") if CONFIG_MOTHERDUCK_API_KEY in config: @@ -320,7 +310,7 @@ def sql_processor( configured_catalog=configured_catalogue, schema_name=test_schema_name, db_path=path, - motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY] + motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY], ) else: processor = destination._get_sql_processor( @@ -349,13 +339,7 @@ def test_write( generator = destination.write( config, configured_catalogue, - [ - airbyte_message1, - airbyte_message2, - airbyte_message3, - airbyte_message4, - airbyte_message5 - ], + [airbyte_message1, airbyte_message2, airbyte_message3, airbyte_message4, airbyte_message5], ) result = list(generator) @@ -407,9 +391,7 @@ def test_write_dupe( assert sql_result[1][1] == "777-54-0664" -def _airbyte_messages( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) @@ -431,9 +413,7 @@ def _airbyte_messages( yield message -def _airbyte_messages_with_inconsistent_json_fields( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) random.seed(0) @@ -453,25 +433,19 @@ def _airbyte_messages_with_inconsistent_json_fields( data=( { "key1": fake.unique.name(), - "key2": str(fake.ssn()) - if random.random() < 0.5 - else str(random.randrange(1000, 9999999999999)), + "key2": str(fake.ssn()) if random.random() < 0.5 else str(random.randrange(1000, 9999999999999)), "nested1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": str(fake.ssn()) - if random.random() < 0.5 - else random.randrange(1000, 9999999999999), + "key4": str(fake.ssn()) if random.random() < 0.5 else random.randrange(1000, 9999999999999), "dictionary1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": "True" - if random.random() < 0.5 - else True, + "key4": "True" if random.random() < 0.5 else True, } ), } @@ -517,16 +491,11 @@ def test_large_number_of_writes( generator = destination.write( config, configured_catalogue, - airbyte_message_generator( - TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name - ), + airbyte_message_generator(TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name), ) result = list(generator) assert len(result) == TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) - sql_result = sql_processor._execute_sql( - "SELECT count(1) " - f"FROM {test_schema_name}.{test_large_table_name}" - ) + sql_result = sql_processor._execute_sql("SELECT count(1) " f"FROM {test_schema_name}.{test_large_table_name}") assert sql_result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-motherduck/main.py b/airbyte-integrations/connectors/destination-motherduck/main.py index 710f67455a47..bc605551741d 100644 --- a/airbyte-integrations/connectors/destination-motherduck/main.py +++ b/airbyte-integrations/connectors/destination-motherduck/main.py @@ -5,5 +5,6 @@ from destination_motherduck.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py index 1245d9785f01..c062b0567053 100644 --- a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py @@ -5,9 +5,10 @@ from unittest.mock import Mock, patch import pytest -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from destination_motherduck.destination import CONFIG_DEFAULT_SCHEMA, DestinationMotherDuck, validated_sql_name +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + def test_validated_sql_name() -> None: assert validated_sql_name("valid_name") == "valid_name" diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py index 67e610934d43..54cc7631fb68 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py @@ -57,13 +57,17 @@ def get_configured_stream_info( ) matching_streams: list[ConfiguredAirbyteStream] = [ - stream for stream in self.configured_catalog.streams if stream.stream.name == stream_name + stream + for stream in self.configured_catalog.streams + if stream.stream.name == stream_name ] if not matching_streams: raise exc.AirbyteStreamNotFoundError( stream_name=stream_name, context={ - "available_streams": [stream.stream.name for stream in self.configured_catalog.streams], + "available_streams": [ + stream.stream.name for stream in self.configured_catalog.streams + ], }, ) diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py index 74bad59eca54..707ed7d4ed0d 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py @@ -16,13 +16,22 @@ from airbyte import exceptions as exc from airbyte.strategies import WriteStrategy -from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, Type +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStreamState, + Type, +) + from destination_pgvector.common.state.state_writers import StateWriterBase, StdOutStateWriter if TYPE_CHECKING: from collections.abc import Iterable, Iterator from airbyte._batch_handles import BatchHandle + from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.common.state.state_writers import StateWriterBase @@ -224,7 +233,9 @@ def process_airbyte_messages( stream_name = record_msg.stream if stream_name not in stream_schemas: - stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema(stream_name=stream_name) + stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema( + stream_name=stream_name + ) self.process_record_message( record_msg, diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py index a297a0194846..9b5af54bf286 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py @@ -21,13 +21,14 @@ from airbyte.strategies import WriteStrategy from airbyte.types import SQLTypeConverter from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode -from destination_pgvector.common.destinations.record_processor import RecordProcessorBase -from destination_pgvector.common.state.state_writers import StdOutStateWriter from pandas import Index from pydantic import BaseModel from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update from sqlalchemy.sql.elements import TextClause +from destination_pgvector.common.destinations.record_processor import RecordProcessorBase +from destination_pgvector.common.state.state_writers import StdOutStateWriter + if TYPE_CHECKING: from collections.abc import Generator @@ -35,14 +36,15 @@ from airbyte._processors.file.base import FileWriterBase from airbyte.secrets.base import SecretString from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage - from destination_pgvector.common.catalog.catalog_providers import CatalogProvider - from destination_pgvector.common.state.state_writers import StateWriterBase from sqlalchemy.engine import Connection, Engine from sqlalchemy.engine.cursor import CursorResult from sqlalchemy.engine.reflection import Inspector from sqlalchemy.sql.base import Executable from sqlalchemy.sql.type_api import TypeEngine + from destination_pgvector.common.catalog.catalog_providers import CatalogProvider + from destination_pgvector.common.state.state_writers import StateWriterBase + class RecordDedupeMode(enum.Enum): APPEND = "append" @@ -101,7 +103,9 @@ def get_vendor_client(self) -> object: Raises `NotImplementedError` if a custom vendor client is not defined. """ - raise NotImplementedError(f"The type '{type(self).__name__}' does not define a custom client.") + raise NotImplementedError( + f"The type '{type(self).__name__}' does not define a custom client." + ) class SqlProcessorBase(RecordProcessorBase): @@ -265,7 +269,9 @@ def _get_table_by_name( query. To ignore the cache and force a refresh, set 'force_refresh' to True. """ if force_refresh and shallow_okay: - raise exc.PyAirbyteInternalError(message="Cannot force refresh and use shallow query at the same time.") + raise exc.PyAirbyteInternalError( + message="Cannot force refresh and use shallow query at the same time." + ) if force_refresh and table_name in self._cached_table_definitions: self._invalidate_table_cache(table_name) @@ -306,7 +312,9 @@ def _ensure_schema_exists( if DEBUG_MODE: found_schemas = self._get_schemas_list() - assert schema_name in found_schemas, f"Schema {schema_name} was not created. Found: {found_schemas}" + assert schema_name in found_schemas, ( + f"Schema {schema_name} was not created. Found: {found_schemas}" + ) def _quote_identifier(self, identifier: str) -> str: """Return the given identifier, quoted.""" @@ -365,7 +373,8 @@ def _get_schemas_list( return [ found_schema.split(".")[-1].strip('"') for found_schema in found_schemas - if "." not in found_schema or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) + if "." not in found_schema + or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) ] def _ensure_final_table_exists( @@ -522,7 +531,9 @@ def finalizing_batches( Returns a mapping of batch IDs to batch handles, for those processed batches. """ batches_to_finalize: list[BatchHandle] = self.file_writer.get_pending_batches(stream_name) - state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[stream_name].copy() + state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[ + stream_name + ].copy() self._pending_state_messages[stream_name].clear() progress.log_batches_finalizing(stream_name, len(batches_to_finalize)) @@ -581,7 +592,9 @@ def _write_files_to_new_table( for file_path in files: dataframe = pd.read_json(file_path, lines=True) - sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions(stream_name) + sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions( + stream_name + ) # Remove fields that are not in the schema for col_name in dataframe.columns: @@ -619,7 +632,10 @@ def _add_column_to_table( ) -> None: """Add a column to the given table.""" self._execute_sql( - text(f"ALTER TABLE {self._fully_qualified(table.name)} " f"ADD COLUMN {column_name} {column_type}"), + text( + f"ALTER TABLE {self._fully_qualified(table.name)} " + f"ADD COLUMN {column_name} {column_type}" + ), ) def _add_missing_columns_to_table( @@ -677,7 +693,9 @@ def _write_temp_table_to_final_table( ) if write_strategy == WriteStrategy.AUTO: - configured_destination_sync_mode: DestinationSyncMode = self.catalog_provider.get_destination_sync_mode(stream_name) + configured_destination_sync_mode: DestinationSyncMode = ( + self.catalog_provider.get_destination_sync_mode(stream_name) + ) if configured_destination_sync_mode == DestinationSyncMode.overwrite: write_strategy = WriteStrategy.REPLACE elif configured_destination_sync_mode == DestinationSyncMode.append: @@ -802,13 +820,11 @@ def _swap_temp_table_with_final_table( _ = stream_name deletion_name = f"{final_table_name}_deleteme" - commands = "\n".join( - [ - f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", - f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " f"TO {final_table_name};", - f"DROP TABLE {self._fully_qualified(deletion_name)};", - ] - ) + commands = "\n".join([ + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME TO {final_table_name};", + f"DROP TABLE {self._fully_qualified(deletion_name)};", + ]) self._execute_sql(commands) def _merge_temp_table_to_final_table( @@ -882,16 +898,23 @@ def _emulated_merge_temp_table_to_final_table( temp_table = self._get_table_by_name(temp_table_name) pk_columns = self._get_primary_keys(stream_name) - columns_to_update: set[str] = self._get_sql_column_definitions(stream_name=stream_name).keys() - set(pk_columns) + columns_to_update: set[str] = self._get_sql_column_definitions( + stream_name=stream_name + ).keys() - set(pk_columns) # Create a dictionary mapping columns in users_final to users_stage for updating update_values = { - self._get_column_by_name(final_table, column): (self._get_column_by_name(temp_table, column)) for column in columns_to_update + self._get_column_by_name(final_table, column): ( + self._get_column_by_name(temp_table, column) + ) + for column in columns_to_update } # Craft the WHERE clause for composite primary keys join_conditions = [ - self._get_column_by_name(final_table, pk_column) == self._get_column_by_name(temp_table, pk_column) for pk_column in pk_columns + self._get_column_by_name(final_table, pk_column) + == self._get_column_by_name(temp_table, pk_column) + for pk_column in pk_columns ] join_clause = and_(*join_conditions) @@ -906,7 +929,9 @@ def _emulated_merge_temp_table_to_final_table( where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null() # Select records from temp_table that are not in final_table - select_new_records_stmt = select([temp_table]).select_from(joined_table).where(where_not_exists_clause) + select_new_records_stmt = ( + select([temp_table]).select_from(joined_table).where(where_not_exists_clause) + ) # Craft the INSERT statement using the select statement insert_new_records_stmt = insert(final_table).from_select( diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py index d3e72198d82e..eb6ebdc51c38 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py @@ -22,7 +22,6 @@ class Config: class PGVectorIndexingModel(BaseModel): - host: str = Field( ..., title="Host", diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py index 32ae5291efdf..44a6abbb7b2c 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py @@ -19,6 +19,7 @@ DestinationSyncMode, Status, ) + from destination_pgvector import pgvector_processor from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.config import ConfigModel @@ -29,7 +30,9 @@ class DestinationPGVector(Destination): sql_processor: pgvector_processor.PGVectorProcessor - def _init_sql_processor(self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None) -> None: + def _init_sql_processor( + self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None + ) -> None: self.sql_processor = pgvector_processor.PGVectorProcessor( sql_config=pgvector_processor.PostgresConfig( host=config.indexing.host, @@ -67,7 +70,9 @@ def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionS self.sql_processor.sql_config.connect() return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") + return AirbyteConnectionStatus( + status=Status.FAILED, message=f"An exception occurred: {repr(e)}" + ) def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: return ConnectorSpecification( diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py index 5757951e05d1..5a33f25ff59b 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py @@ -13,16 +13,27 @@ from airbyte._processors.file.jsonl import JsonlWriter from airbyte.secrets import SecretString from airbyte_cdk.destinations.vector_db_based import embedder -from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor as DocumentSplitter -from airbyte_cdk.destinations.vector_db_based.document_processor import ProcessingConfigModel as DocumentSplitterConfig +from airbyte_cdk.destinations.vector_db_based.document_processor import ( + DocumentProcessor as DocumentSplitter, +) +from airbyte_cdk.destinations.vector_db_based.document_processor import ( + ProcessingConfigModel as DocumentSplitterConfig, +) from airbyte_cdk.models import AirbyteRecordMessage -from destination_pgvector.common.catalog.catalog_providers import CatalogProvider -from destination_pgvector.common.sql.sql_processor import SqlConfig, SqlProcessorBase -from destination_pgvector.globals import CHUNK_ID_COLUMN, DOCUMENT_CONTENT_COLUMN, DOCUMENT_ID_COLUMN, EMBEDDING_COLUMN, METADATA_COLUMN from overrides import overrides from pgvector.sqlalchemy import Vector from typing_extensions import Protocol +from destination_pgvector.common.catalog.catalog_providers import CatalogProvider +from destination_pgvector.common.sql.sql_processor import SqlConfig, SqlProcessorBase +from destination_pgvector.globals import ( + CHUNK_ID_COLUMN, + DOCUMENT_CONTENT_COLUMN, + DOCUMENT_ID_COLUMN, + EMBEDDING_COLUMN, + METADATA_COLUMN, +) + class PostgresConfig(SqlConfig): """Configuration for the Postgres cache. @@ -39,7 +50,9 @@ class PostgresConfig(SqlConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" - return SecretString(f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}") + return SecretString( + f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + ) @overrides def get_database_name(self) -> str: @@ -123,7 +136,9 @@ def _emulated_merge_temp_table_to_final_table( So instead of using UPDATE and then INSERT, we will DELETE all rows for included primary keys and then call the append implementation to insert new rows. """ - columns_list: list[str] = list(self._get_sql_column_definitions(stream_name=stream_name).keys()) + columns_list: list[str] = list( + self._get_sql_column_definitions(stream_name=stream_name).keys() + ) delete_statement = dedent( f""" diff --git a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py index def5bb298801..2046e0945847 100644 --- a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py @@ -10,11 +10,11 @@ import psycopg2 from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest from airbyte_cdk.models import DestinationSyncMode, Status + from destination_pgvector.destination import DestinationPGVector class PGVectorIntegrationTest(BaseIntegrationTest): - def setUp(self): with open("secrets/config.json", "r") as f: self.config = json.loads(f.read()) @@ -331,7 +331,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationPGVector() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -358,4 +358,3 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["document_content"] == "Dogs are" assert third_written_record["document_id"] == "Stream_mystream_Key_0" assert third_written_record["document_content"] == "number 0" - diff --git a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py index 0b671c9ff1aa..0b48b176d7ae 100644 --- a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py @@ -8,6 +8,7 @@ from airbyte.strategies import WriteStrategy from airbyte_cdk.models import ConnectorSpecification, Status + from destination_pgvector.config import ConfigModel from destination_pgvector.destination import DestinationPGVector diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py index 38ade8d0834a..1256143155d5 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class PineconeIndexingModel(BaseModel): pinecone_key: str = Field( diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py index 9d199b3ce50b..45af5865a9c6 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py @@ -17,6 +17,7 @@ from destination_pinecone.config import ConfigModel from destination_pinecone.indexer import PineconeIndexer + BATCH_SIZE = 32 diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py index b2d67010192e..8777829993cf 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py @@ -7,14 +7,16 @@ from typing import Optional import urllib3 +from pinecone import PineconeException +from pinecone.grpc import PineconeGRPC + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception from airbyte_cdk.models import AirbyteConnectionStatus, Status from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog, DestinationSyncMode from destination_pinecone.config import PineconeIndexingModel -from pinecone import PineconeException -from pinecone.grpc import PineconeGRPC + # large enough to speed up processing, small enough to not hit pinecone request limits PINECONE_BATCH_SIZE = 40 diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py index 46215e878464..b51c3ba18131 100644 --- a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py @@ -7,6 +7,13 @@ import os import time +from destination_pinecone.destination import DestinationPinecone +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Pinecone +from pinecone import Pinecone as PineconeREST +from pinecone import PineconeException +from pinecone.grpc import PineconeGRPC + from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest from airbyte_cdk.models import ( @@ -21,12 +28,6 @@ SyncMode, Type, ) -from destination_pinecone.destination import DestinationPinecone -from langchain.embeddings import OpenAIEmbeddings -from langchain.vectorstores import Pinecone -from pinecone import Pinecone as PineconeREST -from pinecone import PineconeException -from pinecone.grpc import PineconeGRPC class PineconeIntegrationTest(BaseIntegrationTest): @@ -35,14 +36,14 @@ def _init_pinecone(self): self.pinecone_index = self.pc.Index(self.config["indexing"]["index"]) self.pc_rest = PineconeREST(api_key=self.config["indexing"]["pinecone_key"]) self.pinecone_index_rest = self.pc_rest.Index(name=self.config["indexing"]["index"]) - + def _wait(self): - print("Waiting for Pinecone...", end='', flush=True) + print("Waiting for Pinecone...", end="", flush=True) for i in range(15): time.sleep(1) - print(".", end='', flush=True) + print(".", end="", flush=True) print() # Move to the next line after the loop - + def setUp(self): with open("secrets/config.json", "r") as f: self.config = json.loads(f.read()) @@ -50,28 +51,28 @@ def setUp(self): def tearDown(self): self._wait() - # make sure pinecone is initialized correctly before cleaning up + # make sure pinecone is initialized correctly before cleaning up self._init_pinecone() try: self.pinecone_index.delete(delete_all=True) except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in default namespace. No data in the index/namespace.") try: self.pinecone_index.delete(delete_all=True, namespace="ns1") except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in ns1 namespace. No data in the index/namespace.") def test_integration_test_flag_is_set(self): assert "PYTEST_CURRENT_TEST" in os.environ def test_check_valid_config(self): - outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) + outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): @@ -88,7 +89,7 @@ def test_check_invalid_config(self): }, }, ) - + assert outcome.status == Status.FAILED def test_write(self): @@ -99,21 +100,20 @@ def test_write(self): # initial sync destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - - - self._wait() + + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 # incrementalally update a doc incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) - - self._wait() - + + self._wait() + result = self.pinecone_index.query( vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_ab_record_id": "mystream_2"}, include_metadata=True ) - + assert len(result.matches) == 1 assert ( result.matches[0].metadata["text"] == "str_col: Cats are nice" @@ -135,19 +135,21 @@ def test_write_with_namespace(self): destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - self._wait() + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 - def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}} + stream_schema = { + "type": "object", + "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}, + } overwrite_stream = ConfiguredAirbyteStream( stream=AirbyteStream( - name="mystream", + name="mystream", namespace="ns1", - json_schema=stream_schema, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + json_schema=stream_schema, + supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], ), primary_key=[["int_col"]], sync_mode=SyncMode.incremental, @@ -155,14 +157,9 @@ def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSy ) return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) - + def _record_with_namespace(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, - namespace="ns1", - data={"str_col": str_value, "int_col": int_value}, - emitted_at=0) + type=Type.RECORD, + record=AirbyteRecordMessage(stream=stream, namespace="ns1", data={"str_col": str_value, "int_col": int_value}, emitted_at=0), ) - - - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-pinecone/main.py b/airbyte-integrations/connectors/destination-pinecone/main.py index 9a12601a2cfa..fdc7a3e63099 100644 --- a/airbyte-integrations/connectors/destination-pinecone/main.py +++ b/airbyte-integrations/connectors/destination-pinecone/main.py @@ -7,5 +7,6 @@ from destination_pinecone import DestinationPinecone + if __name__ == "__main__": DestinationPinecone().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py index 3145b0a66e8c..bff3f91507c1 100644 --- a/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py +++ b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py @@ -10,6 +10,7 @@ from langchain.llms import OpenAI from langchain.vectorstores import Pinecone + # Run with OPENAI_API_KEY, PINECONE_KEY and PINECONE_ENV set in the environment embeddings = OpenAIEmbeddings() diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py index e62c0bb038e9..3a0329deee08 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_pinecone.config import ConfigModel from destination_pinecone.destination import DestinationPinecone +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationPinecone(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py index 5d4b714a7902..3fb46fa23a8f 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py @@ -7,23 +7,24 @@ import pytest import urllib3 -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_pinecone.config import PineconeIndexingModel from destination_pinecone.indexer import PineconeIndexer from pinecone import IndexDescription, exceptions from pinecone.grpc import PineconeGRPC from pinecone.models import IndexList +from airbyte_cdk.models import ConfiguredAirbyteCatalog + def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") - with patch.object(PineconeGRPC, 'Index') as mock_index: + with patch.object(PineconeGRPC, "Index") as mock_index: indexer = PineconeIndexer(config, 3) - + indexer.pc.list_indexes = MagicMock() indexer.pc.list_indexes.return_value.indexes = create_mock_list_indexes() - + indexer.pc.describe_index = MagicMock() if side_effect: indexer.pc.describe_index.side_effect = side_effect @@ -31,6 +32,7 @@ def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): indexer.pc.describe_index.return_value = create_index_description(dimensions=embedding_dimensions) return indexer + def create_index_description(dimensions=3, pod_type="p1"): return IndexDescription( name="", @@ -41,18 +43,21 @@ def create_index_description(dimensions=3, pod_type="p1"): status=None, ) + def create_mock_list_indexes(): return [{"name": "myindex"}, {"name": "myindex2"}] + @pytest.fixture(scope="module", autouse=True) def mock_describe_index(): with patch("pinecone.describe_index") as mock: mock.return_value = create_index_description() yield mock + @pytest.fixture(scope="module", autouse=True) def mock_determine_spec_type(): - with patch.object(PineconeIndexer, 'determine_spec_type') as mock: + with patch.object(PineconeIndexer, "determine_spec_type") as mock: mock.return_value = "pod" yield mock @@ -77,7 +82,7 @@ def test_get_source_tag_with_pytest(): @patch.dict("os.environ", {"RUN_IN_AIRBYTE_CI": "Value does not matter"}) def test_get_source_tag_with_ci(): - # CI and pytest is running + # CI and pytest is running indexer = create_pinecone_indexer() assert indexer.get_source_tag() == "airbyte_test" @@ -143,6 +148,7 @@ def test_pinecone_index_upsert_and_delete_starter(mock_describe_index, mock_dete namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "pod" @@ -160,9 +166,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(filter={'_ab_record_id': {'$in': ['delete_id1', 'delete_id2']}}, namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}}, namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -173,6 +177,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "serverless" @@ -190,9 +195,7 @@ def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_d "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(ids=['delete_id1', 'delete_id2'], namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(ids=["delete_id1", "delete_id2"], namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -311,13 +314,12 @@ def test_pinecone_pre_sync_starter(mock_describe_index, mock_determine_spec_type ("myindex", None, 3, True, None), ("other_index", None, 3, False, "Index other_index does not exist in environment"), ( - "myindex", + "myindex", urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve 'controller.myenv.pinecone.io'")), 3, False, "Failed to resolve environment", - - ), + ), ("myindex", exceptions.UnauthorizedException(http_resp=urllib3.HTTPResponse(body="No entry!")), 3, False, "No entry!"), ("myindex", None, 4, False, "Make sure embedding and indexing configurations match."), ("myindex", Exception("describe failed"), 3, False, "describe failed"), diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py index e877a93bcf6d..31102ed2d858 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py @@ -5,9 +5,10 @@ from typing import Literal, Union -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic.v1 import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class NoAuth(BaseModel): mode: Literal["no_auth"] = Field("no_auth", const=True) diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py index 8f2756c17b20..bde7a6791044 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py @@ -15,6 +15,7 @@ from destination_qdrant.config import ConfigModel from destination_qdrant.indexer import QdrantIndexer + BATCH_SIZE = 256 @@ -29,7 +30,6 @@ def _init_indexer(self, config: ConfigModel): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - config_model = ConfigModel.parse_obj(config) self._init_indexer(config_model) writer = Writer( @@ -48,7 +48,6 @@ def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCon return AirbyteConnectionStatus(status=Status.SUCCEEDED) def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: - return ConnectorSpecification( documentationUrl="https://docs.airbyte.com/integrations/destinations/qdrant", supportsIncremental=True, diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py index 1d78d8730e7a..40292a02f991 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py @@ -6,15 +6,17 @@ import uuid from typing import List, Optional +from qdrant_client import QdrantClient, models +from qdrant_client.conversions.common_types import PointsSelector +from qdrant_client.models import Distance, PayloadSchemaType, VectorParams + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog, Level, Type from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from destination_qdrant.config import QdrantIndexingConfigModel -from qdrant_client import QdrantClient, models -from qdrant_client.conversions.common_types import PointsSelector -from qdrant_client.models import Distance, PayloadSchemaType, VectorParams + DISTANCE_METRIC_MAP = { "dot": Distance.DOT, diff --git a/airbyte-integrations/connectors/destination-qdrant/main.py b/airbyte-integrations/connectors/destination-qdrant/main.py index 42c2e8492e9f..003ce287d263 100644 --- a/airbyte-integrations/connectors/destination-qdrant/main.py +++ b/airbyte-integrations/connectors/destination-qdrant/main.py @@ -7,5 +7,6 @@ from destination_qdrant import DestinationQdrant + if __name__ == "__main__": DestinationQdrant().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py index ec3cb89da4c2..7ffd02511229 100644 --- a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_qdrant.config import ConfigModel from destination_qdrant.destination import DestinationQdrant +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationQdrant(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py index eef6619302aa..34c83e028b05 100644 --- a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py @@ -5,12 +5,13 @@ import unittest from unittest.mock import Mock, call -from airbyte_cdk.destinations.vector_db_based.utils import format_exception -from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, AirbyteStream, DestinationSyncMode, Level, SyncMode, Type from destination_qdrant.config import QdrantIndexingConfigModel from destination_qdrant.indexer import QdrantIndexer from qdrant_client import models +from airbyte_cdk.destinations.vector_db_based.utils import format_exception +from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, AirbyteStream, DestinationSyncMode, Level, SyncMode, Type + class TestQdrantIndexer(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py b/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py index 5a7512f1ae14..ff26ffef0e49 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py @@ -8,11 +8,13 @@ from typing import Any, Iterable, Mapping import pika -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from pika.adapters.blocking_connection import BlockingConnection from pika.spec import BasicProperties +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + + _DEFAULT_PORT = 5672 diff --git a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py index f99c64178d4f..edff22328506 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py @@ -5,6 +5,8 @@ import json from unittest.mock import Mock +from destination_rabbitmq.destination import DestinationRabbitmq, create_connection + from airbyte_cdk.models import AirbyteMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, @@ -15,7 +17,7 @@ DestinationSyncMode, SyncMode, ) -from destination_rabbitmq.destination import DestinationRabbitmq, create_connection + TEST_STREAM = "animals" TEST_NAMESPACE = "test_namespace" diff --git a/airbyte-integrations/connectors/destination-rabbitmq/main.py b/airbyte-integrations/connectors/destination-rabbitmq/main.py index fc09374015c7..cb9d8cbf44df 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/main.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/main.py @@ -7,5 +7,6 @@ from destination_rabbitmq import DestinationRabbitmq + if __name__ == "__main__": DestinationRabbitmq().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py index 57c34b6f9f58..23c4d6feeae4 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py @@ -7,6 +7,9 @@ from unittest import mock from unittest.mock import Mock +from destination_rabbitmq.destination import DestinationRabbitmq +from pika.spec import Queue + from airbyte_cdk.models import AirbyteMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, @@ -17,8 +20,7 @@ DestinationSyncMode, SyncMode, ) -from destination_rabbitmq.destination import DestinationRabbitmq -from pika.spec import Queue + config = { "host": "test.rabbitmq", diff --git a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py index a08452ebc2bd..87a07ad189c2 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py +++ b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py @@ -20,7 +20,6 @@ def write( configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage], ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py index a78f9b391aa6..3053118858ef 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py @@ -9,6 +9,9 @@ import docker import pytest +from destination_sftp_json import DestinationSftpJson +from destination_sftp_json.client import SftpClient + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import ( AirbyteMessage, @@ -22,8 +25,6 @@ SyncMode, Type, ) -from destination_sftp_json import DestinationSftpJson -from destination_sftp_json.client import SftpClient @pytest.fixture(scope="module") diff --git a/airbyte-integrations/connectors/destination-sftp-json/main.py b/airbyte-integrations/connectors/destination-sftp-json/main.py index 84167715983d..6bcd0cbf1368 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/main.py +++ b/airbyte-integrations/connectors/destination-sftp-json/main.py @@ -7,5 +7,6 @@ from destination_sftp_json import DestinationSftpJson + if __name__ == "__main__": DestinationSftpJson().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-sftp-json/setup.py b/airbyte-integrations/connectors/destination-sftp-json/setup.py index 1ade4c13054a..00c14d35802d 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/setup.py +++ b/airbyte-integrations/connectors/destination-sftp-json/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = ["airbyte-cdk", "smart_open==5.1.0", "paramiko==2.10.1"] TEST_REQUIREMENTS = ["pytest~=6.1", "docker==5.0.3"] diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py index 2867def1a4a9..cdf0670f7dd2 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py @@ -330,9 +330,9 @@ def _ensure_schema_exists( if DEBUG_MODE: found_schemas = self._get_schemas_list() - assert ( - schema_name in found_schemas - ), f"Schema {schema_name} was not created. Found: {found_schemas}" + assert schema_name in found_schemas, ( + f"Schema {schema_name} was not created. Found: {found_schemas}" + ) def _quote_identifier(self, identifier: str) -> str: """Return the given identifier, quoted.""" @@ -839,9 +839,8 @@ def _swap_temp_table_with_final_table( _ = stream_name deletion_name = f"{final_table_name}_deleteme" commands = "\n".join([ - f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", - f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " - f"TO {final_table_name};", + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME TO {final_table_name};", f"DROP TABLE {self._fully_qualified(deletion_name)};", ]) self._execute_sql(commands) diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py index 769eb67fc984..3a514b274173 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py @@ -7,7 +7,6 @@ import abc from typing import TYPE_CHECKING - if TYPE_CHECKING: from airbyte_protocol.models.airbyte_protocol import AirbyteStateMessage diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py index 3148de0862d1..17a602da056b 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py @@ -3,7 +3,6 @@ from __future__ import annotations - DOCUMENT_ID_COLUMN = "document_id" CHUNK_ID_COLUMN = "chunk_id" METADATA_COLUMN = "metadata" diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py index 864c227d2868..1976d878dae7 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py @@ -337,7 +337,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationSnowflakeCortex() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -364,8 +364,6 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["DOCUMENT_CONTENT"] == '"Dogs are"' assert third_written_record["DOCUMENT_ID"] == "Stream_mystream_Key_0" assert third_written_record["DOCUMENT_CONTENT"] == '"number 0"' - - """ Following tests are not code specific, but are useful to confirm that the Cortex functions are available and behaving as expcected diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py index e5595ecf8891..a89fb499be9c 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py @@ -2,11 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import unittest from unittest.mock import MagicMock, Mock, patch from airbyte.strategies import WriteStrategy -import logging from airbyte_cdk.models import ConnectorSpecification, Status from destination_snowflake_cortex.config import ConfigModel diff --git a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py index f8049536e7b4..c1466fbf1ba6 100644 --- a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py +++ b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py @@ -37,7 +37,6 @@ def _get_destination_path(destination_path: str) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. @@ -65,9 +64,7 @@ def write( # delete the tables query = """ DROP TABLE IF EXISTS {} - """.format( - table_name - ) + """.format(table_name) con.execute(query) # create the table if needed query = """ @@ -76,9 +73,7 @@ def write( _airbyte_emitted_at TEXT, _airbyte_data TEXT ) - """.format( - table_name=table_name - ) + """.format(table_name=table_name) con.execute(query) buffer = defaultdict(list) @@ -87,13 +82,10 @@ def write( if message.type == Type.STATE: # flush the buffer for stream_name in buffer.keys(): - query = """ INSERT INTO {table_name} VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) + """.format(table_name=f"_airbyte_raw_{stream_name}") con.executemany(query, buffer[stream_name]) @@ -113,13 +105,10 @@ def write( # flush any remaining messages for stream_name in buffer.keys(): - query = """ INSERT INTO {table_name} VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) + """.format(table_name=f"_airbyte_raw_{stream_name}") con.executemany(query, buffer[stream_name]) diff --git a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py index ed5d04c99eda..52ea5544e5ba 100644 --- a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py @@ -12,6 +12,8 @@ from unittest.mock import MagicMock import pytest +from destination_sqlite import DestinationSqlite + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -23,7 +25,6 @@ SyncMode, Type, ) -from destination_sqlite import DestinationSqlite @pytest.fixture(autouse=True) diff --git a/airbyte-integrations/connectors/destination-sqlite/main.py b/airbyte-integrations/connectors/destination-sqlite/main.py index 8f641827f84c..c3aabc20b451 100644 --- a/airbyte-integrations/connectors/destination-sqlite/main.py +++ b/airbyte-integrations/connectors/destination-sqlite/main.py @@ -7,5 +7,6 @@ from destination_sqlite import DestinationSqlite + if __name__ == "__main__": DestinationSqlite().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py index 6cffbfc15897..2fbfb3b87172 100644 --- a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py @@ -7,6 +7,8 @@ from logging import getLogger from typing import Any, Iterable, Mapping +from timeplus import Environment, Stream + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -17,7 +19,7 @@ Status, Type, ) -from timeplus import Environment, Stream + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py index e3de7dac9e71..b231cdd0f47d 100755 --- a/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py @@ -8,6 +8,8 @@ from typing import Any, Mapping import pytest +from destination_timeplus import DestinationTimeplus + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,7 +21,6 @@ SyncMode, Type, ) -from destination_timeplus import DestinationTimeplus @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-timeplus/main.py b/airbyte-integrations/connectors/destination-timeplus/main.py index a6f1b6b49d3c..5c1e78fefd90 100755 --- a/airbyte-integrations/connectors/destination-timeplus/main.py +++ b/airbyte-integrations/connectors/destination-timeplus/main.py @@ -7,5 +7,6 @@ from destination_timeplus import DestinationTimeplus + if __name__ == "__main__": DestinationTimeplus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py index fbbf027e2e28..8a629a6a4a94 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -7,10 +7,11 @@ import time from typing import Any, Iterable, Mapping +from typesense import Client + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from destination_typesense.writer import TypesenseWriter -from typesense import Client def get_client(config: Mapping[str, Any]) -> Client: diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index 54e85d5512b7..bedaaf76fd2a 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -9,6 +9,7 @@ from typesense import Client + logger = getLogger("airbyte") @@ -35,6 +36,6 @@ def flush(self): for stream, data in self.write_buffer: grouped_by_stream[stream].append(data) - for (stream, data) in grouped_by_stream.items(): + for stream, data in grouped_by_stream.items(): self.client.collections[stream].documents.import_(data) self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py index a57b7cc59fd2..bc0302eee8d9 100644 --- a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from typing import Any, Dict, Mapping import pytest +from destination_typesense.destination import DestinationTypesense, get_client +from typesense import Client + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_typesense.destination import DestinationTypesense, get_client -from typesense import Client @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-typesense/main.py b/airbyte-integrations/connectors/destination-typesense/main.py index f702d9f4c0b9..518595701e80 100644 --- a/airbyte-integrations/connectors/destination-typesense/main.py +++ b/airbyte-integrations/connectors/destination-typesense/main.py @@ -7,5 +7,6 @@ from destination_typesense import DestinationTypesense + if __name__ == "__main__": DestinationTypesense().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py index 755d30014780..a46a518bea0f 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py @@ -10,8 +10,10 @@ import backoff import requests + from destination_vectara.config import VectaraConfig + METADATA_STREAM_FIELD = "_ab_stream" @@ -25,7 +27,6 @@ def user_error(e: Exception) -> bool: class VectaraClient: - BASE_URL = "https://api.vectara.io/v1" def __init__(self, config: VectaraConfig): @@ -99,7 +100,6 @@ def _get_jwt_token(self): @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) def _request(self, endpoint: str, http_method: str = "POST", params: Mapping[str, Any] = None, data: Mapping[str, Any] = None): - url = f"{self.BASE_URL}/{endpoint}" current_ts = datetime.datetime.now().timestamp() diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py index 86ca2dba16f5..94cb2b16c418 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py @@ -4,9 +4,10 @@ from typing import List, Optional -from airbyte_cdk.utils.spec_schema_transformations import resolve_refs from pydantic import BaseModel, Field +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs + class OAuth2(BaseModel): client_id: str = Field(..., title="OAuth Client ID", description="OAuth2.0 client id", order=0) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py index b324865d36ba..3c60ed65b96c 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py @@ -24,7 +24,6 @@ class DestinationVectara(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py index 401d279294f0..9c681b8896ee 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py @@ -6,16 +6,17 @@ from typing import Any, Dict, List, Mapping, Optional import dpath.util + from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType from destination_vectara.client import VectaraClient + METADATA_STREAM_FIELD = "_ab_stream" class VectaraWriter: - write_buffer: List[Mapping[str, Any]] = [] flush_interval = 1000 diff --git a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py index 052006303d85..ba9d1aff89e9 100644 --- a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py @@ -7,6 +7,9 @@ import unittest from typing import Any, Dict +from destination_vectara.client import VectaraClient +from destination_vectara.destination import DestinationVectara + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_vectara.client import VectaraClient -from destination_vectara.destination import DestinationVectara class VectaraIntegrationTest(unittest.TestCase): @@ -45,6 +46,7 @@ def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) ) + def _clean(self): self._client.delete_doc_by_metadata(metadata_field_name="_ab_stream", metadata_field_values=["None_mystream"]) diff --git a/airbyte-integrations/connectors/destination-vectara/main.py b/airbyte-integrations/connectors/destination-vectara/main.py index 289b411fb318..81502c9cd22b 100644 --- a/airbyte-integrations/connectors/destination-vectara/main.py +++ b/airbyte-integrations/connectors/destination-vectara/main.py @@ -7,5 +7,6 @@ from destination_vectara import DestinationVectara + if __name__ == "__main__": DestinationVectara().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py index c4708d59ffc9..92d3171a229e 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py @@ -4,6 +4,8 @@ from typing import List, Literal, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import ( AzureOpenAIEmbeddingConfigModel, CohereEmbeddingConfigModel, @@ -14,7 +16,6 @@ VectorDBConfigModel, ) from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic import BaseModel, Field class UsernamePasswordAuth(BaseModel): diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py index 93adb9d825a4..664997753077 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py @@ -12,6 +12,7 @@ from typing import Optional import weaviate + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py index e76a8afe4457..d6d3233f6ba2 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py @@ -8,14 +8,16 @@ import docker import weaviate -from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_weaviate.destination import DestinationWeaviate from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Weaviate from pytest_docker.plugin import get_docker_ip +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status + + WEAVIATE_CONTAINER_NAME = "weaviate-test-container-will-get-deleted" diff --git a/airbyte-integrations/connectors/destination-weaviate/main.py b/airbyte-integrations/connectors/destination-weaviate/main.py index 83b1692b9225..2b3cc1b28456 100644 --- a/airbyte-integrations/connectors/destination-weaviate/main.py +++ b/airbyte-integrations/connectors/destination-weaviate/main.py @@ -7,5 +7,6 @@ from destination_weaviate import DestinationWeaviate + if __name__ == "__main__": DestinationWeaviate().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py index 6d16b865f31c..474bef7e3e86 100644 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_weaviate.config import ConfigModel from destination_weaviate.destination import DestinationWeaviate +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationWeaviate(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py index a5b2526e392c..270a78608bae 100644 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py @@ -6,11 +6,12 @@ from collections import defaultdict from unittest.mock import ANY, Mock, call, patch -from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode from destination_weaviate.config import NoAuth, TokenAuth, WeaviateIndexingConfigModel from destination_weaviate.indexer import WeaviateIndexer, WeaviatePartialBatchError +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode + class TestWeaviateIndexer(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py index 461110bfa284..688f35cc1906 100644 --- a/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py +++ b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py @@ -5,11 +5,13 @@ import logging from typing import Any, Iterable, Mapping -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from xata.client import XataClient from xata.helpers import BulkProcessor +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + + __version__ = "0.0.1" logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py index b98d151d31d3..326ff59b8e94 100644 --- a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from unittest.mock import Mock import pytest +from destination_xata import DestinationXata +from xata.client import XataClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -18,8 +21,6 @@ SyncMode, Type, ) -from destination_xata import DestinationXata -from xata.client import XataClient @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-xata/main.py b/airbyte-integrations/connectors/destination-xata/main.py index 76e7d8f087c0..d195eab47321 100644 --- a/airbyte-integrations/connectors/destination-xata/main.py +++ b/airbyte-integrations/connectors/destination-xata/main.py @@ -7,5 +7,6 @@ from destination_xata import DestinationXata + if __name__ == "__main__": DestinationXata().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-adjust/main.py b/airbyte-integrations/connectors/source-adjust/main.py index a361a2be887c..6dc1a06978ec 100644 --- a/airbyte-integrations/connectors/source-adjust/main.py +++ b/airbyte-integrations/connectors/source-adjust/main.py @@ -4,5 +4,6 @@ from source_adjust.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/components.py b/airbyte-integrations/connectors/source-adjust/source_adjust/components.py index 141b701274e9..1cb725a98c71 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/components.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/components.py @@ -11,7 +11,6 @@ @dataclass class AdjustSchemaLoader(JsonFileSchemaLoader): - config: Mapping[str, Any] def get_json_schema(self) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/model.py b/airbyte-integrations/connectors/source-adjust/source_adjust/model.py index 849f8cb5dc3f..cb47761929be 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/model.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/model.py @@ -15,6 +15,7 @@ import pydantic + BASE_METRICS = typing.Literal[ "network_cost", "network_cost_diff", diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/source.py b/airbyte-integrations/connectors/source-adjust/source_adjust/source.py index 01ec2350059d..a07ed0ad520a 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/source.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py index 1930c55800b8..91a0d2ce6911 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py @@ -12,7 +12,7 @@ def config_pass(): "metrics": ["installs", "network_installs", "network_cost", "network_ecpi"], "dimensions": ["app", "partner_name", "campaign", "campaign_id_network", "campaign_network"], "additional_metrics": [], - "until_today": True + "until_today": True, } @@ -31,11 +31,7 @@ def mock_report_response(): return { "rows": [ { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,14 +40,9 @@ def mock_report_response(): "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } ], - "totals": { - "installs": 10.0, - "network_installs": 0.0, - "network_cost": 0.0, - "network_ecpi": 0.0 - }, + "totals": {"installs": 10.0, "network_installs": 0.0, "network_cost": 0.0, "network_ecpi": 0.0}, "warnings": [], } diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py index 909d26ac70ee..4d3dc022aaa4 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py @@ -5,9 +5,10 @@ from datetime import datetime, timedelta from typing import Any, Mapping +from source_adjust.source import SourceAdjust + from airbyte_cdk.sources.streams import Stream from airbyte_protocol.models import SyncMode -from source_adjust.source import SourceAdjust def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py index 081ab6d9c05b..cd7a13745507 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py @@ -5,11 +5,12 @@ from datetime import datetime from typing import Any, Mapping -from airbyte_cdk.sources.streams import Stream -from airbyte_protocol.models import SyncMode from jsonref import requests from source_adjust.source import SourceAdjust +from airbyte_cdk.sources.streams import Stream +from airbyte_protocol.models import SyncMode + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceAdjust() @@ -31,11 +32,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp requests_mock.get(url=report_url, status_code=200, json=mock_report_response) stream = get_stream_by_name("AdjustReport", config_pass) expected_parsed_record = { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,7 +41,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } records = [] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-airtable/main.py b/airbyte-integrations/connectors/source-airtable/main.py index 170d6caf75b1..61d016c6e10f 100644 --- a/airbyte-integrations/connectors/source-airtable/main.py +++ b/airbyte-integrations/connectors/source-airtable/main.py @@ -4,5 +4,6 @@ from source_airtable.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py index 3cf295b20562..a4283a121240 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py @@ -4,6 +4,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py index 7cde021f9552..336e8be7b7eb 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py @@ -7,6 +7,7 @@ from typing import Mapping, Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py index 7ae780d39d15..0a29d79dfcdd 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + AIRTABLE_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 403: ErrorResolution( response_action=ResponseAction.FAIL, diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py index 692e09ecae40..f1517a456785 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py @@ -5,6 +5,7 @@ from typing import Any, Mapping, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py index 5ec6f022df9f..f70fbacb499e 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py @@ -9,11 +9,11 @@ from airbyte_cdk.models import AirbyteStream from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode + logger: logging.Logger = logging.getLogger("airbyte") class SchemaTypes: - string: Dict = {"type": ["null", "string"]} number: Dict = {"type": ["null", "number"]} diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py index 655959c751c0..8d841efd64f9 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py @@ -18,7 +18,6 @@ class SourceAirtable(AbstractSource): - logger: logging.Logger = logging.getLogger("airbyte") streams_catalog: Iterable[Mapping[str, Any]] = [] _auth: AirtableAuth = None diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py index 39e985b948fa..5a1ddeb60845 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpClient, HttpStream from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator @@ -15,6 +16,7 @@ from source_airtable.airtable_error_handler import AirtableErrorHandler from source_airtable.schema_helpers import SchemaHelpers + URL_BASE: str = "https://api.airtable.com/v0/" @@ -97,7 +99,6 @@ def path(self, **kwargs) -> str: class AirtableStream(HttpStream, ABC): def __init__(self, stream_path: str, stream_name: str, stream_schema, table_name: str, **kwargs): - self.stream_name = stream_name self.stream_path = stream_path self.stream_schema = stream_schema diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py index f2a42ae96efb..161a0529852b 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py @@ -4,10 +4,11 @@ import pytest +from source_airtable.streams import AirtableStream + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_airtable.streams import AirtableStream @pytest.fixture diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py index e9ee73055239..3e0ae06c53b8 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py @@ -10,13 +10,7 @@ from source_airtable.airtable_backoff_strategy import AirtableBackoffStrategy -@pytest.mark.parametrize( - "response_code, expected_backoff_time", - [ - (429, 30), - (404, None) - ] -) +@pytest.mark.parametrize("response_code, expected_backoff_time", [(429, 30), (404, None)]) def test_backoff_time(response_code, expected_backoff_time): mocked_logger = MagicMock(spec=logging.Logger) backoff = AirtableBackoffStrategy(logger=mocked_logger) diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py index 67daf8eb3129..f10fe1cfe951 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py @@ -6,21 +6,30 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from requests import Response from source_airtable.airtable_error_handler import AirtableErrorHandler from source_airtable.airtable_error_mapping import AIRTABLE_ERROR_MAPPING from source_airtable.auth import AirtableOAuth +from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.mark.parametrize( "auth, json_response, error_message", [ - (TokenAuthenticator, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable"), - (AirtableOAuth, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Access Token does not have required permissions, please reauthenticate."), - (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable.") - ] + ( + TokenAuthenticator, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable", + ), + ( + AirtableOAuth, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Access Token does not have required permissions, please reauthenticate.", + ), + (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable."), + ], ) def test_interpret_response_handles_403_error(auth, json_response, error_message): mocked_authenticator = MagicMock(spec=auth) @@ -35,6 +44,7 @@ def test_interpret_response_handles_403_error(auth, json_response, error_message assert error_resolution.failure_type == FailureType.config_error assert error_resolution.error_message == error_message + def test_interpret_response_defers_to_airtable_error_mapping_for_other_errors(): mocked_logger = MagicMock(spec=logging.Logger) mocked_response = MagicMock(spec=Response) diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py index fbc0c6e9e49f..766779eeddab 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py @@ -4,9 +4,11 @@ import pytest +from source_airtable.auth import AirtableAuth, AirtableOAuth + from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils import AirbyteTracedException -from source_airtable.auth import AirtableAuth, AirtableOAuth + CONFIG_OAUTH = {"credentials": {"auth_method": "oauth2.0", "client_id": "sample_client_id", "client_secret": "sample_client_secret"}} diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py index d1591ab7fbb0..eab9bed768a2 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py @@ -7,9 +7,10 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import AirbyteCatalog from source_airtable.source import SourceAirtable +from airbyte_cdk.models import AirbyteCatalog + @pytest.mark.parametrize( "status, check_passed", diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py index e9a8d96dd37a..c31d0982e866 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py @@ -11,7 +11,6 @@ class TestBases: - bases_instance = AirtableBases(authenticator=MagicMock()) def test_url_base(self): @@ -50,7 +49,6 @@ def test_parse_response(self, fake_bases_response, expected_bases_response, requ class TestTables: - tables_instance = AirtableTables(base_id="test_base_id", authenticator=MagicMock()) def test_path(self): diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/main.py b/airbyte-integrations/connectors/source-alpha-vantage/main.py index dcccfe7a535c..48d5bd87d7f3 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/main.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/main.py @@ -4,5 +4,6 @@ from source_alpha_vantage.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py index fa00fb61a53d..91d0018826b2 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -7,6 +7,7 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py index fe5a71ac01fb..e865183827a2 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_alpha_vantage import SourceAlphaVantage +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAlphaVantage() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py index e8cede80087d..7cec796ec207 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index 30a0b6957860..c85eb27fa106 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -4,5 +4,6 @@ from source_amazon_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py index 33233bb6448d..e5bebdddd0ba 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py index fd69c06fe9d1..6782766e81c6 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -8,10 +8,11 @@ import traceback from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_amazon_ads import SourceAmazonAds from source_amazon_ads.config_migrations import MigrateStartDate diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 5b714f9f41ee..8d41e57a5eea 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk import TState from airbyte_cdk.models import AdvancedAuth, AuthFlowType, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource @@ -16,6 +17,7 @@ from .spec import SourceAmazonAdsSpec from .streams import Profiles, SponsoredBrandsV3ReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream + # Oauth 2.0 authentication URL for amazon TOKEN_URL = "https://api.amazon.com/auth/o2/token" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 3156ac42e5a4..555e17e9c584 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -5,9 +5,10 @@ from enum import Enum from typing import List, Optional -from airbyte_cdk.sources.config import BaseConfig from pydantic.v1 import Field +from airbyte_cdk.sources.config import BaseConfig + class StateFilterEnum(str, Enum): enabled = "enabled" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 1c9aa88ff121..3ed493d99d67 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -7,13 +7,15 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import requests +from pydantic.v1 import BaseModel, ValidationError + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction -from pydantic.v1 import BaseModel, ValidationError from source_amazon_ads.constants import URL_MAPPING + """ This class hierarchy may seem overcomplicated so here is a visualization of class to provide explanation why it had been done in this way. diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py index f1896621bbca..a8917baf8c12 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py @@ -5,6 +5,7 @@ from typing import Any, Iterable, List, Mapping import requests + from airbyte_cdk.models import SyncMode from source_amazon_ads.streams.common import AmazonAdsStream diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py index 8b28b64b46b8..8887c3ccaf6f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py @@ -6,6 +6,7 @@ from .products_report import SponsoredProductsReportStream + METRICS_MAP_V3 = { "purchasedAsin": [ "campaignBudgetCurrencyCode", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py index fbe3742bc707..428958c58a3d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py @@ -6,10 +6,12 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from source_amazon_ads.streams.report_streams.report_stream_models import ReportInfo from source_amazon_ads.streams.report_streams.report_streams import ReportStream + METRICS_MAP_V3 = { "campaigns": [ "addToCart", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index e616d58573a9..c8a532c0d0fa 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -7,10 +7,12 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from .report_streams import ReportInfo, ReportStream + METRICS_MAP = { "campaigns": [ "campaignName", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index d3369fcfbf03..1f87d37ffb8a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -15,11 +15,12 @@ import backoff import pendulum import requests +from pendulum import Date + from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from pendulum import Date from source_amazon_ads.streams.common import BasicAmazonAdsStream from source_amazon_ads.utils import get_typed_env, iterate_one_by_one diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py index 62444e6984e0..5cde2adfd521 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py @@ -6,6 +6,7 @@ import os from typing import Union + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py index b17aeca0e4e4..d448a1ad9681 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py @@ -5,6 +5,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional + BRAND_REFERRAL_BONUS = "brb_bonus_amount" METRICS_MAP = { diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py index a9a989b8d8c7..2acee74d842c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py @@ -5,6 +5,7 @@ from airbyte_cdk.test.mock_http.response_builder import find_template + CLIENT_ID = "amzn.app-oa2-client.test" CLIENT_SECRET = "test-secret" REGION = "NA" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py index d97869cd5d9b..4bee406a893a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py @@ -6,6 +6,7 @@ from zoneinfo import ZoneInfo import freezegun + from airbyte_cdk.models import Level as LogLevel from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker @@ -17,6 +18,7 @@ from .config import ConfigBuilder from .utils import get_log_messages_by_log_level, read_stream + REPORTING_PERIOD = 90 _NOW = datetime.now(timezone.utc) _A_START_DATE = _NOW - timedelta(days=REPORTING_PERIOD) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py index 660623df5daf..fe0d429ada68 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py @@ -6,11 +6,11 @@ import pendulum import requests_mock -from airbyte_cdk.models import AirbyteStateBlob +from source_amazon_ads.streams.report_streams import brands_report, display_report, products_report + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker, HttpRequestMatcher -from source_amazon_ads.streams.report_streams import brands_report, display_report, products_report from .ad_requests import ( OAuthRequestBuilder, diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py index 16f8849fdcfa..96874e60649b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py @@ -25,6 +25,7 @@ from .config import ConfigBuilder from .utils import get_log_messages_by_log_level, read_stream + _DEFAULT_REQUEST_BODY = json.dumps({"maxResults": 100}) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py index c81c14347aa1..3e91740f9f3f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py @@ -3,12 +3,12 @@ import operator from typing import Any, Dict, List, Optional -from airbyte_cdk.models import AirbyteMessage +from source_amazon_ads import SourceAmazonAds + +from airbyte_cdk.models import AirbyteMessage, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from source_amazon_ads import SourceAmazonAds def read_stream( diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index b810621b9eff..831e3e53303b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -13,8 +13,6 @@ import pytest import requests_mock import responses -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode from freezegun import freeze_time from pendulum import Date from pytest import raises @@ -23,8 +21,12 @@ from source_amazon_ads.streams.report_streams.report_stream_models import RecordType from source_amazon_ads.streams.report_streams.report_streams import ReportGenerationFailure, ReportGenerationInProgress, TooManyRequests +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode + from .utils import read_incremental + """ METRIC_RESPONSE is gzip compressed binary representing this string: [ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index eb6cbb4dd93f..a993909cc455 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -6,12 +6,13 @@ from unittest import mock from urllib.parse import urlparse, urlunparse +from source_amazon_ads.config_migrations import MigrateStartDate + from airbyte_cdk.models import SyncMode from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config -from source_amazon_ads.config_migrations import MigrateStartDate def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index ee7f33aa3ce5..e9aadc718c43 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -4,5 +4,6 @@ from source_amazon_seller_partner.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 4bdaab19df35..d062e1d85046 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -6,6 +6,7 @@ from typing import Any, Mapping import pendulum + from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py index 39a7055af1b7..b98903bd9863 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceAmazonSellerPartner + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py index 4b0e1e99bbe5..618a808ba0c3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py @@ -31,6 +31,7 @@ Australia A39IBJ37TRP1C6 AU Japan A1VC38T7YXB528 JP """ + from enum import Enum from typing import Dict, Tuple diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index d2e1fda1a0c0..623e103127c7 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -7,12 +7,13 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from requests import HTTPError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment from airbyte_protocol.models import ConnectorSpecification -from requests import HTTPError from source_amazon_seller_partner.auth import AWSAuthenticator from source_amazon_seller_partner.constants import get_marketplaces from source_amazon_seller_partner.streams import ( @@ -73,6 +74,7 @@ ) from source_amazon_seller_partner.utils import AmazonConfigException + # given the retention period: 730 DEFAULT_RETENTION_PERIOD_IN_DAYS = 730 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index b65b7eb9a845..a414590399a0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -17,6 +17,7 @@ import pendulum import requests import xmltodict + from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin, package_name_from_class @@ -28,6 +29,7 @@ from airbyte_protocol.models import FailureType from source_amazon_seller_partner.utils import STREAM_THRESHOLD_PERIOD, threshold_period_decorator + REPORTS_API_VERSION = "2021-06-30" ORDERS_API_VERSION = "v0" VENDORS_API_VERSION = "v1" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py index 6c340ecfe566..7696b235dae6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py @@ -7,6 +7,7 @@ from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType + LOG_LEVEL = logging.getLevelName("INFO") LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py index 12579417383d..6165eea8d437 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py @@ -8,6 +8,7 @@ import pytest + os.environ["DEPLOYMENT_MODE"] = "testing" @@ -25,10 +26,7 @@ def init_kwargs() -> Dict[str, Any]: @pytest.fixture def report_init_kwargs(init_kwargs) -> Dict[str, Any]: - return { - "stream_name": "GET_TEST_REPORT", - **init_kwargs - } + return {"stream_name": "GET_TEST_REPORT", **init_kwargs} @pytest.fixture diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py index 75cccd73e380..05c8b5c47d34 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + ACCESS_TOKEN = "test_access_token" LWA_APP_ID = "amazon_app_id" LWA_CLIENT_SECRET = "amazon_client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py index 79b1528ed038..7d7a549dcc53 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py @@ -7,6 +7,7 @@ from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_TOKEN_STRING = "MDAwMDAwMDAwMQ==" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index 1225aa30b181..1a23c7c99549 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -11,17 +11,19 @@ import freezegun import pytest import requests_mock +from source_amazon_seller_partner.streams import ReportProcessingStatus + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode -from source_amazon_seller_partner.streams import ReportProcessingStatus from .config import CONFIG_END_DATE, CONFIG_START_DATE, MARKETPLACE_ID, NOW, VENDOR_TRAFFIC_REPORT_CONFIG_END_DATE, ConfigBuilder from .request_builder import RequestBuilder from .response_builder import build_response, response_with_status from .utils import assert_message_in_log_output, config, find_template, get_stream_by_name, mock_auth, read_output + _DOCUMENT_DOWNLOAD_URL = "https://test.com/download" _REPORT_ID = "6789087632" _REPORT_DOCUMENT_ID = "report_document_id" @@ -447,14 +449,16 @@ def test_given_http_error_500_on_create_report_when_read_then_no_records_and_err @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) @HttpMocker() def test_given_http_error_not_support_account_id_of_type_vendor_when_read_then_no_records_and_error_logged( - self, stream_name: str, data_format: str, http_mocker: HttpMocker + self, stream_name: str, data_format: str, http_mocker: HttpMocker ): mock_auth(http_mocker) response_body = { "errors": [ - {"code": "InvalidInput", - "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", - "details": ""} + { + "code": "InvalidInput", + "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", + "details": "", + } ] } http_mocker.post( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py index f3eccb1b2615..35f2a10b7ec8 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py @@ -8,6 +8,7 @@ import freezegun import pendulum + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -28,6 +29,7 @@ from .response_builder import response_with_status from .utils import config, mock_auth, read_output + _START_DATE = pendulum.datetime(year=2023, month=1, day=1) _END_DATE = pendulum.datetime(year=2023, month=1, day=5) _REPLICATION_START_FIELD = "createdAfter" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py index a1416a505328..3937e7fc0873 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py @@ -8,6 +8,7 @@ import freezegun import pendulum + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -28,6 +29,7 @@ from .response_builder import response_with_status from .utils import config, mock_auth, read_output + _START_DATE = pendulum.datetime(year=2023, month=1, day=1) _END_DATE = pendulum.datetime(year=2023, month=1, day=5) _REPLICATION_START_FIELD = "changedAfter" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py index 099f99f42f39..91b7f7c511d2 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py @@ -7,13 +7,14 @@ from http import HTTPStatus from typing import Any, List, Mapping, Optional +from source_amazon_seller_partner import SourceAmazonSellerPartner + from airbyte_cdk.sources.streams import Stream from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import _get_unit_test_folder from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, Level, SyncMode -from source_amazon_seller_partner import SourceAmazonSellerPartner from .config import ACCESS_TOKEN, ConfigBuilder from .request_builder import RequestBuilder diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py index d57b88a9f907..16b924b9c800 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py @@ -7,9 +7,10 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from source_amazon_seller_partner.streams import AnalyticsStream, IncrementalAnalyticsStream +from airbyte_cdk.models import SyncMode + class SomeAnalyticsStream(AnalyticsStream): report_name = "GET_ANALYTICS_STREAM" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index 385699cc5844..46360c3737d6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -8,10 +8,12 @@ import pendulum import pytest import requests +from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents, RestockInventoryReports + from airbyte_cdk.models import SyncMode from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType -from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents, RestockInventoryReports + list_financial_event_groups_data = { "payload": { @@ -211,15 +213,7 @@ def test_reports_read_records_raise_on_backoff(mocker, requests_mock, caplog): requests_mock.post( "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, ) stream = RestockInventoryReports( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index 058dfc514d77..533c96df2e5d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -7,11 +7,13 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions, MigrateStreamNameOption from source_amazon_seller_partner.source import SourceAmazonSellerPartner +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + CMD = "check" SOURCE: Source = SourceAmazonSellerPartner() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index 2d50e3adcb9f..afd6f6e3c1eb 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -4,9 +4,11 @@ import pytest -from airbyte_cdk.models import SyncMode from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports +from airbyte_cdk.models import SyncMode + + START_DATE_1 = "2022-05-25T00:00:00Z" END_DATE_1 = "2022-05-26T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index a77cf2bd0b07..1bc7f5dab48f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -7,11 +7,13 @@ from unittest.mock import patch import pytest -from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner from source_amazon_seller_partner.streams import VendorOrders from source_amazon_seller_partner.utils import AmazonConfigException +from airbyte_cdk.sources.streams import Stream + + logger = logging.getLogger("airbyte") @@ -124,19 +126,23 @@ def test_check_connection_with_orders(requests_mock, connector_config_with_repor ( "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ - ("GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + ( + "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ {"option_name": "some_name_1", "option_value": "some_value_1"}, {"option_name": "some_name_2", "option_value": "some_value_2"}, ], - ), - ] + ), + ], ), ("SOME_OTHER_STREAM", []), ), ) def test_get_stream_report_options_list(connector_config_with_report_options, report_name, stream_name_w_options): - assert list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) == stream_name_w_options + assert ( + list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) + == stream_name_w_options + ) def test_config_report_options_validation_error_duplicated_streams(connector_config_with_report_options): @@ -199,7 +205,9 @@ def test_spec(deployment_mode, common_streams_count, monkeypatch): "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", "GET_VENDOR_TRAFFIC_REPORT", } - streams_with_report_options = SourceAmazonSellerPartner().spec( - logger - ).connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + streams_with_report_options = ( + SourceAmazonSellerPartner() + .spec(logger) + .connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + ) assert len(set(streams_with_report_options).intersection(oss_only_streams)) == common_streams_count diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index cae628254c61..8d5857ec491e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -8,9 +8,6 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType from source_amazon_seller_partner.streams import ( IncrementalReportsAmazonSPStream, ReportProcessingStatus, @@ -18,6 +15,10 @@ VendorDirectFulfillmentShipping, ) +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + class SomeReportStream(ReportsAmazonSPStream): report_name = "GET_TEST_REPORT" @@ -246,15 +247,7 @@ def test_given_429_when_read_records_then_raise_transient_error(self, report_ini "POST", "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, reason="Forbidden", ) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/main.py b/airbyte-integrations/connectors/source-amazon-sqs/main.py index 3e218a144f8f..3d9850f97f6d 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/main.py @@ -4,5 +4,6 @@ from source_amazon_sqs.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index e39e0d894b21..3ba1991d471a 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = ["airbyte-cdk", "boto3"] TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "moto[sqs, iam]"] diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py index 6472649a9b9e..4aedc4e71d9c 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py @@ -8,6 +8,8 @@ from typing import Dict, Generator import boto3 +from botocore.exceptions import ClientError + from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ( AirbyteCatalog, @@ -20,7 +22,6 @@ Type, ) from airbyte_cdk.sources.source import Source -from botocore.exceptions import ClientError class SourceAmazonSqs(Source): diff --git a/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py index 28ead98ff99c..511b2081c087 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py @@ -6,14 +6,15 @@ from typing import Any, Dict, Mapping import boto3 -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Status # from airbyte_cdk.sources.source import Source from moto import mock_iam, mock_sqs from moto.core import set_initial_no_auth_action_count from source_amazon_sqs import SourceAmazonSqs +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Status + @mock_iam def create_user_with_all_permissions(): diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py index cb2230f94ead..c0b6f7e64019 100755 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py @@ -7,9 +7,10 @@ from pathlib import Path import pytest +from source_amplitude.source import SourceAmplitude + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_amplitude.source import SourceAmplitude @pytest.fixture(scope="module") diff --git a/airbyte-integrations/connectors/source-amplitude/main.py b/airbyte-integrations/connectors/source-amplitude/main.py index 14500e9c73e6..66dbbbcd80bf 100644 --- a/airbyte-integrations/connectors/source-amplitude/main.py +++ b/airbyte-integrations/connectors/source-amplitude/main.py @@ -4,5 +4,6 @@ from source_amplitude.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py index 5fa407664b6e..562a5e957bb2 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py @@ -12,10 +12,12 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.types import Config, Record + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py index 3649e5d9b311..12b0d3fe1614 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_amplitude import SourceAmplitude +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAmplitude() diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py index 6ff2160da69c..cc98c73367b5 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py @@ -5,10 +5,12 @@ from base64 import b64encode from typing import Any, List, Mapping +from source_amplitude.streams import Events + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_amplitude.streams import Events + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py index d6f22fa668d2..40112fa5e0ff 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py @@ -12,6 +12,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream @@ -19,6 +20,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction + LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py index 7d78a625301a..560b7248eabe 100644 --- a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py @@ -11,11 +11,12 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from source_amplitude.components import ActiveUsersRecordExtractor, AverageSessionLengthRecordExtractor, EventsExtractor from source_amplitude.streams import Events +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + @pytest.mark.parametrize( "custom_extractor, data, expected", @@ -141,9 +142,9 @@ def test_event_read(self, requests_mock): "error_code, expectation", [ (400, pytest.raises(AirbyteTracedException)), - (404, does_not_raise()), # does not raise because response action is IGNORE + (404, does_not_raise()), # does not raise because response action is IGNORE (504, pytest.raises(AirbyteTracedException)), - (500, does_not_raise()), # does not raise because repsonse action is RETRY + (500, does_not_raise()), # does not raise because repsonse action is RETRY ], ) def test_event_errors_read(self, mocker, requests_mock, error_code, expectation): @@ -195,7 +196,6 @@ def test_events_parse_response(self, file_name, content_is_valid, records_count) assert len(parsed_response) == records_count if content_is_valid: - # RFC3339 pattern pattern = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$" diff --git a/airbyte-integrations/connectors/source-apify-dataset/components.py b/airbyte-integrations/connectors/source-apify-dataset/components.py index 0fea713e975a..2b1710ddd800 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/components.py +++ b/airbyte-integrations/connectors/source-apify-dataset/components.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appsflyer/main.py b/airbyte-integrations/connectors/source-appsflyer/main.py index ebf2655b2ec4..ba1145064175 100644 --- a/airbyte-integrations/connectors/source-appsflyer/main.py +++ b/airbyte-integrations/connectors/source-appsflyer/main.py @@ -4,5 +4,6 @@ from source_appsflyer.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py index 8e77d5b78019..90cf565ac9c5 100644 --- a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py +++ b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py @@ -12,12 +12,13 @@ import pendulum import requests +from pendulum.tz.timezone import Timezone + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum.tz.timezone import Timezone from .fields import * diff --git a/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py index cc6dae86af52..6251fd0352f6 100644 --- a/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py @@ -4,7 +4,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from pytest import fixture, raises from source_appsflyer.fields import * from source_appsflyer.source import ( @@ -22,6 +21,8 @@ UninstallEvents, ) +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-asana/main.py b/airbyte-integrations/connectors/source-asana/main.py index 5fde4e3c72d6..ade635df308f 100644 --- a/airbyte-integrations/connectors/source-asana/main.py +++ b/airbyte-integrations/connectors/source-asana/main.py @@ -4,5 +4,6 @@ from source_asana.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-asana/source_asana/components.py b/airbyte-integrations/connectors/source-asana/source_asana/components.py index 41595bc273bd..528d372c222b 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/components.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/components.py @@ -7,10 +7,11 @@ from pkgutil import get_data from typing import Any, Mapping, MutableMapping, Optional, Union +from yaml import safe_load + from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState -from yaml import safe_load @dataclass diff --git a/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py b/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py index 64c2ff3945eb..60ae8c42cb5a 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-asana/source_asana/run.py b/airbyte-integrations/connectors/source-asana/source_asana/run.py index fb05d363e6e0..221a9a071af1 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/run.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/run.py @@ -8,11 +8,12 @@ from datetime import datetime from typing import List -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type from orjson import orjson from source_asana import SourceAsana +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type + from .config_migration import AsanaConfigMigration diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 22b0872e587b..4e4efe46b638 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py index 56efb7c45d89..4bb360ca2e2a 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py @@ -5,18 +5,23 @@ from source_asana.config_migration import AsanaConfigMigration from source_asana.source import SourceAsana + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" def test_should_migrate(): assert AsanaConfigMigration.should_migrate({"access_token": "asdfcxz"}) is True - assert AsanaConfigMigration.should_migrate({"credentials": { "option_title": "PAT Credentials", "personal_access_token": "1206938133417909" }} -) is False + assert ( + AsanaConfigMigration.should_migrate( + {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "1206938133417909"}} + ) + is False + ) def test__modify_and_save(): user_config = {"access_token": "asdfcxz"} - expected = {"credentials": { "option_title": "PAT Credentials", "personal_access_token": "asdfcxz" }} + expected = {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "asdfcxz"}} # todo: need to make the migrate a classmethod instead of staticmethod since the missing config field will fail validation source = SourceAsana(config=user_config, catalog=None, state=None) diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-avni/main.py b/airbyte-integrations/connectors/source-avni/main.py index 5ab8e86addc5..7ff1ad92a8b0 100644 --- a/airbyte-integrations/connectors/source-avni/main.py +++ b/airbyte-integrations/connectors/source-avni/main.py @@ -5,9 +5,11 @@ import sys -from airbyte_cdk.entrypoint import launch from source_avni import SourceAvni +from airbyte_cdk.entrypoint import launch + + if __name__ == "__main__": source = SourceAvni() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py index d47cbf7654f6..782f0f2100ef 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/components.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/components.py @@ -4,6 +4,7 @@ import boto3 import requests + from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator @@ -11,7 +12,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self) -> str: - username = self._username.eval(self.config) password = self._password.eval(self.config) @@ -29,7 +29,6 @@ def auth_header(self) -> str: return "auth-token" def get_client_id(self): - url_client = "https://app.avniproject.org/idp-details" response = requests.get(url_client) response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-avni/source_avni/run.py b/airbyte-integrations/connectors/source-avni/source_avni/run.py index 636ce2134b8d..4b1154b784ae 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/run.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_avni import SourceAvni +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAvni() diff --git a/airbyte-integrations/connectors/source-avni/source_avni/source.py b/airbyte-integrations/connectors/source-avni/source_avni/source.py index e6c65ceadb7d..23d6f53b3e51 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/source.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py index 49f77bed58ec..2ac44c6aaf65 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py @@ -7,14 +7,13 @@ from source_avni.components import CustomAuthenticator -@patch('boto3.client') +@patch("boto3.client") def test_token_property(mock_boto3_client): - mock_cognito_client = Mock() mock_boto3_client.return_value = mock_cognito_client - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") source._username = Mock() source._username.eval.return_value = "test_username" source._password = Mock() @@ -22,24 +21,18 @@ def test_token_property(mock_boto3_client): source.get_client_id = Mock() source.get_client_id.return_value = "test_client_id" - mock_cognito_client.initiate_auth.return_value = { - "AuthenticationResult": { - "IdToken": "test_id_token" - } - } + mock_cognito_client.initiate_auth.return_value = {"AuthenticationResult": {"IdToken": "test_id_token"}} token = source.token mock_boto3_client.assert_called_once_with("cognito-idp", region_name="ap-south-1") mock_cognito_client.initiate_auth.assert_called_once_with( - ClientId="test_client_id", - AuthFlow="USER_PASSWORD_AUTH", - AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} + ClientId="test_client_id", AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} ) assert token == "test_id_token" + def test_get_client_id(mocker): - - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") client_id = source.get_client_id() expected_length = 26 - assert len(client_id) == expected_length \ No newline at end of file + assert len(client_id) == expected_length diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py index c2cf082ef31f..b99a2a62f301 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py @@ -10,6 +10,7 @@ from typing import Any, Mapping, Union import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py b/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py index 6f490c8b1b11..bb54e8147922 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py index 62b485cc5303..2f6bd0adfe41 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py @@ -11,15 +11,17 @@ import azure import docker import pytest -from airbyte_protocol.models import ConfiguredAirbyteCatalog from azure.storage.blob import BlobServiceClient, ContainerClient from azure.storage.blob._shared.authentication import SharedKeyCredentialPolicy from fastavro import parse_schema, writer from pandas import read_csv from source_azure_blob_storage import SourceAzureBlobStorage +from airbyte_protocol.models import ConfiguredAirbyteCatalog + from .utils import get_docker_ip, load_config + logger = logging.getLogger("airbyte") JSON_TO_AVRO_TYPES = {"string": "string", "integer": "long", "number": "float", "object": "record"} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py index 29c9ffd1ef1a..7c4b5b24f679 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py @@ -4,10 +4,11 @@ from typing import Any, Mapping import pytest +from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader + from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_protocol.models import ConfiguredAirbyteCatalog -from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/main.py b/airbyte-integrations/connectors/source-azure-blob-storage/main.py index 5d60acbf692a..37897c63affe 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/main.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/main.py @@ -5,5 +5,6 @@ from source_azure_blob_storage.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py index d66169a9d7ae..dd9430d7ee75 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py @@ -9,19 +9,18 @@ from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message + logger = logging.getLogger("airbyte_logger") class MigrateConfig(ABC): @classmethod @abstractmethod - def should_migrate(cls, config: Mapping[str, Any]) -> bool: - ... + def should_migrate(cls, config: Mapping[str, Any]) -> bool: ... @classmethod @abstractmethod - def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: - ... + def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: ... @classmethod def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py index ed331a9c4a2b..51ce990fd215 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util +from pydantic import AnyUrl, BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from pydantic import AnyUrl, BaseModel, Field class Oauth2(BaseModel): diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index a394b0adf581..5a835782741e 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -6,15 +6,16 @@ from typing import Iterable, List, Optional, Union import pytz -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from azure.core.credentials import AccessToken from azure.core.exceptions import ResourceNotFoundError from azure.storage.blob import BlobServiceClient, ContainerClient from smart_open import open +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator + from .spec import SourceAzureBlobStorageSpec diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py index dbe30bfeb7fe..0ff89dec1f18 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py @@ -6,20 +6,19 @@ def test_custom_authenticator(requests_mock): - authenticator = AzureOauth2Authenticator( - token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", - client_id="client_id", - client_secret="client_secret", - refresh_token="refresh_token", - ) + token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + client_id="client_id", + client_secret="client_secret", + refresh_token="refresh_token", + ) token_refresh_response = { "token_type": "Bearer", "scope": "https://storage.azure.com/user_impersonation https://storage.azure.com/.default", "expires_in": 5144, "ext_expires_in": 5144, "access_token": "access_token", - "refresh_token": "refresh_token" + "refresh_token": "refresh_token", } requests_mock.post("https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", json=token_refresh_response) new_token = authenticator.get_token() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py index b41630da48bd..160a7b2bf190 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py @@ -5,10 +5,11 @@ import os from typing import Any, Mapping -from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader from source_azure_blob_storage.config_migrations import MigrateCredentials, MigrateLegacyConfig +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor + # HELPERS def load_config(config_path: str) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py index 3cec8f051aa2..26f3296a6f53 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py @@ -12,24 +12,26 @@ from source_azure_blob_storage.spec import SourceAzureBlobStorageSpec from source_azure_blob_storage.stream_reader import AzureOauth2Authenticator, SourceAzureBlobStorageStreamReader + logger = logging.Logger("") @pytest.mark.parametrize( "credentials, expected_credentials_type", [ - ({"auth_type": "oauth2", - "tenant_id": "tenant_id", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token" - }, AzureOauth2Authenticator), - ({ - "auth_type": "storage_account_key", - "azure_blob_storage_account_key": "key1" - }, str), + ( + { + "auth_type": "oauth2", + "tenant_id": "tenant_id", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, + AzureOauth2Authenticator, + ), + ({"auth_type": "storage_account_key", "azure_blob_storage_account_key": "key1"}, str), ], - ids=["oauth2", "storage_account_key"] + ids=["oauth2", "storage_account_key"], ) def test_stream_reader_credentials(credentials: Dict, expected_credentials_type: Union[str, AzureOauth2Authenticator]): reader = SourceAzureBlobStorageStreamReader() @@ -59,9 +61,9 @@ def test_stream_reader_files_read_and_filter_by_date(): reader.config = config with patch.object(ContainerClient, "list_blobs") as blobs: blobs.return_value = [ - BlobProperties(name='sample_file_1.csv', **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_2.csv', **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_3.csv', **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}) + BlobProperties(name="sample_file_1.csv", **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_2.csv", **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_3.csv", **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}), ] files = list(reader.get_matching_files(globs=["**"], prefix=None, logger=logger)) assert len(files) == 2 diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-table/main.py b/airbyte-integrations/connectors/source-azure-table/main.py index 0831f8065766..0ccc48e59be2 100644 --- a/airbyte-integrations/connectors/source-azure-table/main.py +++ b/airbyte-integrations/connectors/source-azure-table/main.py @@ -4,5 +4,6 @@ from source_azure_table.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py index 752e8472480c..aa1b2aaceb78 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py @@ -23,14 +23,11 @@ def test_get_table_service_client_handles_exception(mocker, reader): """ Test that get_table_service_client method handles exceptions correctly. """ - mocker.patch( - "source_azure_table.azure_table.TableServiceClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableServiceClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_service_client() - + assert "Connection error" in str(exc_info.value) @@ -58,10 +55,7 @@ def test_get_table_client_handles_exception(mocker, reader): reader.get_table_client("") assert "table name is not valid." in str(exc_info.value) - mocker.patch( - "source_azure_table.azure_table.TableClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_client("valid_table_name") @@ -74,10 +68,7 @@ def test_get_tables_return(mocker, reader, tables): """ mock_client = mocker.MagicMock() mock_client.list_tables.return_value = tables.__iter__() - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) result = reader.get_tables() result_table_names = [table.name for table in result] @@ -92,10 +83,7 @@ def test_get_tables_handles_exception(mocker, reader): """ mock_client = mocker.MagicMock() mock_client.list_tables.side_effect = Exception("Failed to list tables") - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) with pytest.raises(Exception) as exc_info: reader.get_tables() diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py index 956375acda0b..d9a455c7def7 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import AirbyteCatalog, SyncMode from source_azure_table.streams import AzureTableStream +from airbyte_cdk.models import AirbyteCatalog, SyncMode + # Tests def test_discover(mocker, config, tables, source, logger): diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bamboo-hr/components.py b/airbyte-integrations/connectors/source-bamboo-hr/components.py index 8035c61dc25d..0f925f3bee74 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/components.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/components.py @@ -11,7 +11,6 @@ @dataclass class CustomReportsSchemaLoader(JsonFileSchemaLoader): - config: Mapping[str, Any] parameters: InitVar[Mapping[str, Any]] = {"name": "custom_reports_stream"} diff --git a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bigcommerce/components.py b/airbyte-integrations/connectors/source-bigcommerce/components.py index d9cdab67a57d..5a823f6da031 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/components.py +++ b/airbyte-integrations/connectors/source-bigcommerce/components.py @@ -7,6 +7,7 @@ import dpath.util import pendulum + from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @@ -20,7 +21,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Record: - kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice} for parsed_field in self._parsed_fields: date_time = parsed_field.value.eval(config, **kwargs) diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bing-ads/main.py b/airbyte-integrations/connectors/source-bing-ads/main.py index c05297b01ad8..dfb7775f3071 100644 --- a/airbyte-integrations/connectors/source-bing-ads/main.py +++ b/airbyte-integrations/connectors/source-bing-ads/main.py @@ -4,5 +4,6 @@ from source_bing_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py index abeb59c2f003..4aed49b85a71 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -7,13 +7,14 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.error import URLError -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream from bingads.service_client import ServiceClient from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager -from source_bing_ads.client import Client from suds import sudsobject +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from source_bing_ads.client import Client + class BingAdsBaseStream(Stream, ABC): primary_key: Optional[Union[str, List[str], List[List[str]]]] = None diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py index 159aba01bf79..85e01f86d693 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py @@ -8,16 +8,16 @@ import pandas as pd import pendulum +from numpy import nan + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from numpy import nan from source_bing_ads.base_streams import Accounts, BingAdsBaseStream from source_bing_ads.utils import transform_bulk_datetime_format_to_rfc_3339 class BingAdsBulkStream(BingAdsBaseStream, IncrementalMixin, ABC): - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) cursor_field = "Modified Time" primary_key = "Id" diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 06c1c801917c..223882405783 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -14,8 +14,6 @@ import backoff import pendulum -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens, OAuthWebAuthCodeGrant from bingads.exceptions import OAuthTokenRequestException from bingads.service_client import ServiceClient @@ -25,6 +23,10 @@ from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from suds import WebFault, sudsobject +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + + FILE_TYPE = "Csv" TIMEOUT_IN_MILLISECONDS = 3_600_000 diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py index dc49ab462cb4..5640921c89da 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import _csv import re import xml.etree.ElementTree as ET from abc import ABC, abstractmethod @@ -9,20 +10,20 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union from urllib.parse import urlparse -import _csv import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from bingads import ServiceClient from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord from bingads.v13.reporting import ReportingDownloadParameters from cached_property import cached_property +from suds import WebFault, sudsobject + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from source_bing_ads.base_streams import Accounts, BingAdsStream from source_bing_ads.utils import transform_date_format_to_rfc_3339, transform_report_hourly_datetime_format_to_rfc_3339 -from suds import WebFault, sudsobject class HourlyReportTransformerMixin: diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py index 153de0086423..219e4628f8f0 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py @@ -5,11 +5,6 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from airbyte_cdk.models import AirbyteStateMessage, SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_cdk.test.mock_http import HttpMocker -from airbyte_cdk.test.state_builder import StateBuilder from bingads.v13.bulk import BulkServiceManager from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from client_builder import build_request, response_with_status @@ -18,6 +13,12 @@ from suds.transport.https import HttpAuthenticated from suds_response_mock import mock_http_authenticated_send +from airbyte_cdk.models import AirbyteStateMessage, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.state_builder import StateBuilder + class BaseTest(TestCase): @property diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py index 1606921d8bd0..3e16e05a03c6 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py @@ -4,6 +4,7 @@ from airbyte_cdk.test.mock_http.response_builder import find_template + TENNANT_ID = "common" DEVELOPER_TOKEN = "test-token" REFRESH_TOKEN = "test-refresh-token" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py index 59b3bea844ea..ed8a64e1842e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py @@ -3,6 +3,7 @@ from suds.transport import Reply, Request from suds.transport.https import HttpAuthenticated + SEARCH_ACCOUNTS_RESPONSE = b""" 6f0a329e-4cb4-4c79-9c08-2dfe601ba05a diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py index d97762fc0bf5..01530c1fad97 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py @@ -2,15 +2,16 @@ from typing import Any, Dict, Optional, Tuple from unittest.mock import MagicMock, patch -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_cdk.test.mock_http import HttpMocker from base_test import BaseTest from source_bing_ads.source import SourceBingAds from suds.transport.https import HttpAuthenticated from suds_response_mock import mock_http_authenticated_send +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker + class TestAccountsStream(BaseTest): stream_name = "accounts" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py index 1dd16c11461f..534dcb61a5bd 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestAppInstallAdLabelsStream(TestBulkStream): stream_name = "app_install_ad_labels" @@ -38,7 +39,9 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-04T12:12:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-04T12:12:12.028+00:00" + } @HttpMocker() @freeze_time("2024-02-26") # mock current time as stream data available for 30 days only @@ -46,14 +49,14 @@ def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ad_labels_state", self.stream_name) self.auth_client(http_mocker) output, service_call_mock = self.read_stream( - self.stream_name, - SyncMode.incremental, - self._config, - "app_install_ad_labels_with_state", - state + self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_state", state ) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-29T12:55:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-29T12:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py index 7524b3241110..913bf6105f00 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestAppInstallAdsStream(TestBulkStream): stream_name = "app_install_ads" @@ -38,16 +39,24 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-03-01T12:49:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-03-01T12:49:12.028+00:00" + } @HttpMocker() @freeze_time("2023-12-29") # mock current time as stream data available for 30 days only def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ads_state", self.stream_name) self.auth_client(http_mocker) - output, service_call_mock = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-01T10:55:12.028+00:00"} + output, service_call_mock = self.read_stream( + self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state + ) + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-01T10:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py index 8c375d58bdc2..6ac874274e0f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestBudgetStream(TestBulkStream): stream_name = "budget" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py index ce40374eed82..fc3bc2c44b8d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py @@ -2,6 +2,7 @@ from test_report_stream import TestSuiteReportStream + FIRST_STATE = {"180535609": {"TimePeriod": "2023-11-12T00:00:00+00:00"}} SECOND_STATE = {"180535609": {"TimePeriod": "2023-11-13T00:00:00+00:00"}} diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py index ae222db7dbd0..7829df1614d2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py @@ -4,12 +4,13 @@ from typing import Any, Optional import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from base_test import BaseTest from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from config_builder import ConfigBuilder +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestReportStream(BaseTest): start_date = "2024-01-01" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py index 9a96becee8c6..7fc12f901ef1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -88,18 +88,18 @@ def test_bulk_stream_read_with_chunks_app_install_ad_labels(mocked_client, confi app_install_ads = AppInstallAdLabels(mocked_client, config) result = app_install_ads.read_with_chunks(path=path_to_file) assert next(result) == { - 'Ad Group': None, - 'Campaign': None, - 'Client Id': 'ClientIdGoesHere', - 'Color': None, - 'Description': None, - 'Id': '-22', - 'Label': None, - 'Modified Time': None, - 'Name': None, - 'Parent Id': '-11112', - 'Status': None, - 'Type': 'App Install Ad Label' + "Ad Group": None, + "Campaign": None, + "Client Id": "ClientIdGoesHere", + "Color": None, + "Description": None, + "Id": "-22", + "Label": None, + "Modified Time": None, + "Name": None, + "Parent Id": "-11112", + "Status": None, + "Type": "App Install Ad Label", } @@ -116,17 +116,21 @@ def test_bulk_stream_read_with_chunks_ioe_error(mocked_client, config, caplog): @pytest.mark.parametrize( "stream_state, config_start_date, expected_start_date", [ - ({"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC)), + ( + {"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, + "2020-01-01", + DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC), + ), ({"another_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", None), ({}, "2020-01-01", None), ({}, "2023-10-21", DateTime(2023, 10, 21, 0, 0, 0, tzinfo=UTC)), ], - ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"] + ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"], ) def test_bulk_stream_start_date(mocked_client, config, stream_state, config_start_date, expected_start_date): mocked_client.reports_start_date = pendulum.parse(config_start_date) if config_start_date else None stream = AppInstallAds(mocked_client, config) - assert expected_start_date == stream.get_start_date(stream_state, 'some_account_id') + assert expected_start_date == stream.get_start_date(stream_state, "some_account_id") @patch.object(source_bing_ads.source, "Client") @@ -140,18 +144,10 @@ def test_bulk_stream_stream_state(mocked_client, config): assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} # stream state saved to connection state stream.state = { - "120342748234": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "27364572345": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "732645723": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "837563864": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - } + "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "27364572345": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "732645723": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "837563864": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, } assert stream.state == { "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, @@ -165,4 +161,6 @@ def test_bulk_stream_stream_state(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_bulk_stream_custom_transform_date_rfc3339(mocked_client, config): stream = AppInstallAds(mocked_client, config) - assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339("04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field]) + assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339( + "04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py index 96b2d5777f57..597c3477f13f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py @@ -10,12 +10,13 @@ import pytest import source_bing_ads.client -from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens from bingads.v13.bulk import BulkServiceManager from bingads.v13.reporting.exceptions import ReportingDownloadException from suds import sudsobject +from airbyte_cdk.utils import AirbyteTracedException + def test_sudsobject_todict_primitive_types(): test_arr = ["1", "test", 1, [0, 0]] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index f68acb43a302..aad542afec40 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import _csv import copy import json import xml.etree.ElementTree as ET @@ -9,11 +10,9 @@ from unittest.mock import MagicMock, Mock, patch from urllib.parse import urlparse -import _csv import pendulum import pytest import source_bing_ads -from airbyte_cdk.models import SyncMode from bingads.service_info import SERVICE_INFO_DICT_V13 from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord, _RowValues @@ -53,6 +52,9 @@ from source_bing_ads.source import SourceBingAds from suds import WebFault +from airbyte_cdk.models import SyncMode + + TEST_CONFIG = { "developer_token": "developer_token", "client_id": "client_id", @@ -259,7 +261,10 @@ def test_report_parse_response_csv_error(caplog): fake_response = MagicMock() fake_response.report_records.__iter__ = MagicMock(side_effect=_csv.Error) list(stream_report.parse_response(fake_response)) - assert "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." in caplog.messages + assert ( + "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." + in caplog.messages + ) @patch.object(source_bing_ads.source, "Client") @@ -404,10 +409,10 @@ def test_account_performance_report_monthly_stream_slices(mocked_client, config_ with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'LastYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'ThisYear'} + {"account_id": 180519267, "customer_id": 100, "time_period": "LastYear"}, + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "LastYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, ] @@ -417,10 +422,7 @@ def test_account_performance_report_monthly_stream_slices_no_time_period(mocked_ accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) - assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100}, - {'account_id': 180278106, 'customer_id': 200} - ] + assert stream_slice == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] @pytest.mark.parametrize( @@ -451,14 +453,38 @@ def test_custom_performance_report_no_last_year_stream_slices(mocked_client, con (AccountPerformanceReportHourly, "hourly_reports/account_performance.csv", "hourly_reports/account_performance_records.json"), (AdGroupPerformanceReportHourly, "hourly_reports/ad_group_performance.csv", "hourly_reports/ad_group_performance_records.json"), (AdPerformanceReportHourly, "hourly_reports/ad_performance.csv", "hourly_reports/ad_performance_records.json"), - (CampaignImpressionPerformanceReportHourly, "hourly_reports/campaign_impression_performance.csv", "hourly_reports/campaign_impression_performance_records.json"), + ( + CampaignImpressionPerformanceReportHourly, + "hourly_reports/campaign_impression_performance.csv", + "hourly_reports/campaign_impression_performance_records.json", + ), (KeywordPerformanceReportHourly, "hourly_reports/keyword_performance.csv", "hourly_reports/keyword_performance_records.json"), - (GeographicPerformanceReportHourly, "hourly_reports/geographic_performance.csv", "hourly_reports/geographic_performance_records.json"), + ( + GeographicPerformanceReportHourly, + "hourly_reports/geographic_performance.csv", + "hourly_reports/geographic_performance_records.json", + ), (AgeGenderAudienceReportHourly, "hourly_reports/age_gender_audience.csv", "hourly_reports/age_gender_audience_records.json"), - (SearchQueryPerformanceReportHourly, "hourly_reports/search_query_performance.csv", "hourly_reports/search_query_performance_records.json"), - (UserLocationPerformanceReportHourly, "hourly_reports/user_location_performance.csv", "hourly_reports/user_location_performance_records.json"), - (AccountImpressionPerformanceReportHourly, "hourly_reports/account_impression_performance.csv", "hourly_reports/account_impression_performance_records.json"), - (AdGroupImpressionPerformanceReportHourly, "hourly_reports/ad_group_impression_performance.csv", "hourly_reports/ad_group_impression_performance_records.json"), + ( + SearchQueryPerformanceReportHourly, + "hourly_reports/search_query_performance.csv", + "hourly_reports/search_query_performance_records.json", + ), + ( + UserLocationPerformanceReportHourly, + "hourly_reports/user_location_performance.csv", + "hourly_reports/user_location_performance_records.json", + ), + ( + AccountImpressionPerformanceReportHourly, + "hourly_reports/account_impression_performance.csv", + "hourly_reports/account_impression_performance_records.json", + ), + ( + AdGroupImpressionPerformanceReportHourly, + "hourly_reports/ad_group_impression_performance.csv", + "hourly_reports/ad_group_impression_performance_records.json", + ), ], ) @patch.object(source_bing_ads.source, "Client") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index a938300eedf0..f53cac5a1080 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -6,12 +6,13 @@ import pytest import source_bing_ads -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from bingads.service_info import SERVICE_INFO_DICT_V13 from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns from source_bing_ads.source import SourceBingAds +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + @patch.object(source_bing_ads.source, "Client") def test_streams_config_based(mocked_client, config): @@ -97,7 +98,6 @@ def test_clear_reporting_object_name(): @patch.object(source_bing_ads.source, "Client") def test_campaigns_request_params(mocked_client, config): - campaigns = Campaigns(mocked_client, config) request_params = campaigns.request_params(stream_slice={"account_id": "account_id"}) @@ -110,7 +110,6 @@ def test_campaigns_request_params(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_campaigns_stream_slices(mocked_client, config): - campaigns = Campaigns(mocked_client, config) accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): @@ -123,7 +122,6 @@ def test_campaigns_stream_slices(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_adgroups_stream_slices(mocked_client, config): - adgroups = AdGroups(mocked_client, config) accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) campaigns_read_records = [iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])] @@ -140,7 +138,6 @@ def test_adgroups_stream_slices(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_ads_request_params(mocked_client, config): - ads = Ads(mocked_client, config) request_params = ads.request_params(stream_slice={"ad_group_id": "ad_group_id"}) @@ -155,7 +152,6 @@ def test_ads_request_params(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_ads_stream_slices(mocked_client, config): - ads = Ads(mocked_client, config) with patch.object( diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-braintree/main.py b/airbyte-integrations/connectors/source-braintree/main.py index d4ae7bec5223..2b5b482a3bcd 100644 --- a/airbyte-integrations/connectors/source-braintree/main.py +++ b/airbyte-integrations/connectors/source-braintree/main.py @@ -4,5 +4,6 @@ from source_braintree.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py index 7ca4cc60abfd..6e8e40f64845 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py @@ -6,10 +6,11 @@ from typing import Any, Dict, Optional, Type import pydantic -from airbyte_cdk.sources.utils.schema_helpers import expand_refs from pydantic import BaseModel from pydantic.typing import resolve_annotations +from airbyte_cdk.sources.utils.schema_helpers import expand_refs + class AllOptional(pydantic.main.ModelMetaclass): """ diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py index 29d30182144c..15d9235267c9 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py @@ -6,9 +6,6 @@ from typing import List, Union import requests -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor -from airbyte_cdk.sources.declarative.types import Record -from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from braintree.attribute_getter import AttributeGetter from braintree.customer import Customer as BCustomer from braintree.discount import Discount as BDiscount @@ -18,8 +15,13 @@ from braintree.subscription import Subscription as BSubscription from braintree.transaction import Transaction as BTransaction from braintree.util.xml_util import XmlUtil + +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from source_braintree.schemas import Customer, Discount, Dispute, MerchantAccount, Plan, Subscription, Transaction + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-braze/main.py b/airbyte-integrations/connectors/source-braze/main.py index 723116b28098..28735b4616de 100644 --- a/airbyte-integrations/connectors/source-braze/main.py +++ b/airbyte-integrations/connectors/source-braze/main.py @@ -4,5 +4,6 @@ from source_braze.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index 43f778382f75..eab5b4e0fd40 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", ] diff --git a/airbyte-integrations/connectors/source-braze/source_braze/components.py b/airbyte-integrations/connectors/source-braze/source_braze/components.py index 06e003ec51d8..0771877d7c63 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/components.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/components.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-braze/source_braze/run.py b/airbyte-integrations/connectors/source-braze/source_braze/run.py index 645b7a31df24..c2b1033fb4df 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/run.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_braze import SourceBraze +from airbyte_cdk.entrypoint import launch + def run(): source = SourceBraze() diff --git a/airbyte-integrations/connectors/source-braze/source_braze/source.py b/airbyte-integrations/connectors/source-braze/source_braze/source.py index df957f37fb67..44e98d1eb706 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/source.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-braze/source_braze/transformations.py b/airbyte-integrations/connectors/source-braze/source_braze/transformations.py index 60a13384e269..bd95bdf646a5 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/transformations.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/transformations.py @@ -6,6 +6,7 @@ from typing import Optional import dpath + from airbyte_cdk.sources.declarative.transformations import AddFields from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py index d478ac686adc..76ff085244e8 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from source_braze import DatetimeIncrementalSyncComponent + from airbyte_cdk.sources.declarative.requesters import RequestOption from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType -from source_braze import DatetimeIncrementalSyncComponent def test_datetime_slicer(): diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py index 2ed02ec26eaa..00314360102a 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from source_braze import TransformToRecordComponent +from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition + def test_string_to_dict_transformation(): """ diff --git a/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-cart/main.py b/airbyte-integrations/connectors/source-cart/main.py index c7f69c914848..53fc51e036f4 100644 --- a/airbyte-integrations/connectors/source-cart/main.py +++ b/airbyte-integrations/connectors/source-cart/main.py @@ -4,5 +4,6 @@ from source_cart.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-cart/source_cart/source.py b/airbyte-integrations/connectors/source-cart/source_cart/source.py index 15903dd2210c..1bf2c0c918d6 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/source.py +++ b/airbyte-integrations/connectors/source-cart/source_cart/source.py @@ -13,12 +13,13 @@ import pendulum import requests +from pendulum.parsing.exceptions import ParserError + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from pendulum.parsing.exceptions import ParserError from .streams import Addresses, CustomersCart, OrderItems, OrderPayments, Orders, OrderStatuses, Products diff --git a/airbyte-integrations/connectors/source-cart/source_cart/streams.py b/airbyte-integrations/connectors/source-cart/source_cart/streams.py index f6e31507f38a..1d5d8a95aafa 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/streams.py +++ b/airbyte-integrations/connectors/source-cart/source_cart/streams.py @@ -9,6 +9,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator @@ -93,7 +94,6 @@ def request_params( class IncrementalCartStream(CartStream, ABC): - state_checkpoint_interval = 1000 cursor_field = "updated_at" diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-chargebee/main.py b/airbyte-integrations/connectors/source-chargebee/main.py index 351ea1590b35..00e55c1473d3 100644 --- a/airbyte-integrations/connectors/source-chargebee/main.py +++ b/airbyte-integrations/connectors/source-chargebee/main.py @@ -4,5 +4,6 @@ from source_chargebee.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py index 0f05242ec89e..5df32652c8c5 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py @@ -155,7 +155,6 @@ def set(self) -> None: @dataclass class IncrementalSingleSliceCursor(DeclarativeCursor): - cursor_field: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py index 40a7f2665c33..d410d2cc2590 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py @@ -8,11 +8,12 @@ from datetime import datetime from typing import List -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type from orjson import orjson from source_chargebee import SourceChargebee +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type + def _get_source(args: List[str]): catalog_path = AirbyteEntrypoint.extract_catalog(args) diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py index b3903ecfc21a..1f6e516955b3 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py index 85f0de928865..65fdee149555 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "site": "ConfigBuilder default site", "site_api_key": "ConfigBuilder default site api key", "start_date": "2023-01-01T06:57:44Z", - "product_catalog": "2.0" + "product_catalog": "2.0", } def with_site(self, site: str) -> "ConfigBuilder": @@ -30,4 +30,4 @@ def with_product_catalog(self, product_catalog: str) -> "ConfigBuilder": return self def build(self) -> Dict[str, Any]: - return self._config \ No newline at end of file + return self._config diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py index 0cf9d9d5a5bc..f5cd872ef7c0 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py @@ -8,4 +8,4 @@ class ChargebeePaginationStrategy(PaginationStrategy): @staticmethod def update(response: Dict[str, Any]) -> None: - response["next_offset"] = "[1707076198000,57873868]" \ No newline at end of file + response["next_offset"] = "[1707076198000,57873868]" diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py index 1b97d9e4d6fb..2429e0af5855 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py @@ -9,7 +9,6 @@ class ChargebeeRequestBuilder: - @classmethod def addon_endpoint(cls, site: str, site_api_key: str) -> "ChargebeeRequestBuilder": return cls("addons", site, site_api_key) @@ -69,7 +68,7 @@ def with_include_deleted(self, include_deleted: bool) -> "ChargebeeRequestBuilde return self def with_created_at_btw(self, created_at_btw: List[int]) -> "ChargebeeRequestBuilder": - self._created_at_btw = f'{created_at_btw}' + self._created_at_btw = f"{created_at_btw}" return self def with_updated_at_btw(self, updated_at_btw: List[int]) -> "ChargebeeRequestBuilder": @@ -97,7 +96,7 @@ def with_limit(self, limit: int) -> "ChargebeeRequestBuilder": return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -117,7 +116,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( @@ -126,8 +127,8 @@ def build(self) -> HttpRequest: headers={"Authorization": f"Basic {base64.b64encode((str(self._site_api_key) + ':').encode('utf-8')).decode('utf-8')}"}, ) -class ChargebeeSubstreamRequestBuilder(ChargebeeRequestBuilder): +class ChargebeeSubstreamRequestBuilder(ChargebeeRequestBuilder): @classmethod def subscription_with_scheduled_changes_endpoint(cls, site: str, site_api_key: str) -> "ChargebeeRequestBuilder": return cls("subscriptions", site, site_api_key) @@ -141,7 +142,7 @@ def with_endpoint_path(self, endpoint_path: str) -> "ChargebeeSubstreamRequestBu return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -161,7 +162,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py index f9163b6be3a8..838fb8d82dc7 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py @@ -10,5 +10,6 @@ def a_response_with_status(status_code: int) -> HttpResponse: return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code) + def a_response_with_status_and_header(status_code: int, header: Mapping[str, str]) -> HttpResponse: - return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) \ No newline at end of file + return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py index 91ee4bd28dec..cf14caaf96ac 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "addon" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,31 +59,27 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() source = _source(catalog=catalog, config=config, state=state) return read(source, config, catalog, state, expecting_exception) + @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -96,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -106,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -120,14 +126,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -171,9 +173,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -189,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -202,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py index 0c0d2288b502..854e4a45a7c9 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "coupon" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +104,14 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +122,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +131,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -186,7 +181,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -202,8 +196,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,7 +208,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py index 18c98133aea8..ffd36b12a924 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "customer" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +128,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +137,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -183,9 +184,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -201,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -214,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py index 92323100d784..3aace07fdf28 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "event" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +104,15 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -171,7 +170,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -187,8 +185,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -200,7 +197,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py index e0fe9d45d38a..48756d17096f 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "hosted_page" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -168,9 +172,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -186,8 +190,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -199,9 +202,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py index ed00c2677e24..a441f7711d1b 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "plan" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,10 +126,7 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -175,7 +176,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py index cf8e8b67164d..d92ec52fedda 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "site_migration_detail" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -35,9 +37,9 @@ _NO_STATE = {} _NOW = datetime.now(timezone.utc) -''' +""" Note that this is a semi-incremental stream and tests will need to be adapated accordingly -''' +""" def _a_request() -> ChargebeeRequestBuilder: @@ -61,22 +63,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -86,7 +84,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -101,8 +98,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -110,26 +106,16 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock @HttpMocker() def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination - http_mocker.get( - _a_request().build(), - _a_response().with_record(_a_record()).with_pagination().build() - ) - http_mocker.get( - _a_request().with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() - ) + http_mocker.get(_a_request().build(), _a_response().with_record(_a_record()).with_pagination().build()) + http_mocker.get(_a_request().with_offset("[1707076198000,57873868]").build(), _a_response().with_record(_a_record()).build()) self._read(_config().with_start_date(self._start_date)) # HTTPMocker ensures call are performed - @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -179,7 +165,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - # Site Migration Detail stream is a semi-incremental stream and therefore state acts differently than typical declarative incremental implementation -- state is updated to most recent cursor value read def setUp(self) -> None: @@ -197,8 +182,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date), _NO_STATE) most_recent_state = output.most_recent_state @@ -209,7 +193,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recent_cursor_value(self, http_mocker: HttpMocker) -> None: state_cursor_value = self._start_date_in_seconds + 1 record_cursor_value = state_cursor_value + 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -219,13 +203,17 @@ def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recen output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) @HttpMocker() - def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated(self, http_mocker: HttpMocker) -> None: + def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated( + self, http_mocker: HttpMocker + ) -> None: state_cursor_value = self._start_date_in_seconds record_cursor_value = self._start_date_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -235,5 +223,7 @@ def test_given_record_returned_with_cursor_value_before_state_record_is_not_read output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) assert len(output.records) == 0 diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py index 7b980fa4fbf8..fb2ff1776ff7 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "subscription" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +128,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +137,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -186,7 +187,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -202,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py index f0ad3b8c6225..cb6d46f8124c 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -20,13 +22,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder, ChargebeeSubstreamRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "subscription_with_scheduled_changes" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -62,7 +64,7 @@ def _a_parent_record() -> RecordBuilder: find_template("subscription", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) @@ -71,31 +73,24 @@ def _a_child_record() -> RecordBuilder: find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) def _a_parent_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _a_child_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription_with_scheduled_changes", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -105,7 +100,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -122,11 +116,15 @@ def test_when_read_then_records_are_extracted(self, http_mocker: HttpMocker) -> http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -137,19 +135,29 @@ def test_given_multiple_parents_when_read_then_fetch_for_each_parent(self, http_ a_parent_id = "a_subscription_test" another_parent_id = "another_subscription_test" - http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(a_parent_id)).with_record(_a_parent_record().with_id(another_parent_id)).build() + _a_parent_response() + .with_record(_a_parent_record().with_id(a_parent_id)) + .with_record(_a_parent_record().with_id(another_parent_id)) + .build(), ) http_mocker.get( - _a_child_request().with_parent_id(a_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(a_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) http_mocker.get( - _a_child_request().with_parent_id(another_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(another_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -161,11 +169,15 @@ def test_when_read_then_primary_key_is_set(self, http_mocker: HttpMocker) -> Non http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -173,15 +185,18 @@ def test_when_read_then_primary_key_is_set(self, http_mocker: HttpMocker) -> Non @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: - parent_id = "subscription_test" http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(400), ) @@ -189,15 +204,18 @@ def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocke @HttpMocker() def test_given_http_status_404_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: - parent_id = "subscription_test" http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(404), ) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py index 8ee8f4a9d341..24527edf0f0c 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "virtual_bank_account" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8))) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,14 +126,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -173,9 +173,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py index 54709f56fed0..4dd6cf059289 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import pytest -from airbyte_cdk.sources.types import Record, StreamSlice from source_chargebee.components import CustomFieldTransformation, IncrementalSingleSliceCursor +from airbyte_cdk.sources.types import Record, StreamSlice + @pytest.mark.parametrize( "record, expected_record", @@ -26,11 +27,12 @@ def test_field_transformation(record, expected_record): transformed_record = transformer.transform(record) assert transformed_record == expected_record + @pytest.mark.parametrize( "record_data, expected", [ ({"pk": 1, "name": "example", "updated_at": 1662459011}, True), - ] + ], ) def test_slicer(record_data, expected): date_time_dict = {"updated_at": 1662459010} @@ -50,24 +52,17 @@ def test_slicer(record_data, expected): @pytest.mark.parametrize( "first_record, second_record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662460000}, - True), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662440000}, - False), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2"}, - False), - ({"pk": 1, "name": "example"}, - {"pk": 2, "name": "example2", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662460000}, True), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662440000}, False), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2"}, False), + ({"pk": 1, "name": "example"}, {"pk": 2, "name": "example2", "updated_at": 1662459010}, True), + ], ) def test_is_greater_than_or_equal(first_record, second_record, expected): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") assert slicer.is_greater_than_or_equal(second_record, first_record) == expected + def test_set_initial_state(): cursor_field = "updated_at" cursor_value = 999999999 @@ -75,18 +70,19 @@ def test_set_initial_state(): slicer.set_initial_state(stream_state={cursor_field: cursor_value}) assert slicer._state[cursor_field] == cursor_value + @pytest.mark.parametrize( "record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, True), + ], ) def test_should_be_synced(record, expected): cursor_field = "updated_at" slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field=cursor_field) assert slicer.should_be_synced(record) == expected + def test_stream_slices(): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") stream_slices_instance = slicer.stream_slices() diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-close-com/main.py b/airbyte-integrations/connectors/source-close-com/main.py index f80e76315939..cbd57dadfa8a 100644 --- a/airbyte-integrations/connectors/source-close-com/main.py +++ b/airbyte-integrations/connectors/source-close-com/main.py @@ -4,5 +4,6 @@ from source_close_com.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py b/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py index 6ce18802ab7e..71ac5b4189ce 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import pendulum + from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py index 2b524cb260e6..24f4a6b5f80a 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py @@ -10,6 +10,7 @@ from urllib.parse import parse_qsl, urlparse import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py index 4dc1b251dd97..4f45b8c93d6f 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commcare/main.py b/airbyte-integrations/connectors/source-commcare/main.py index edd438bde5be..fe2d48f84604 100644 --- a/airbyte-integrations/connectors/source-commcare/main.py +++ b/airbyte-integrations/connectors/source-commcare/main.py @@ -4,5 +4,6 @@ from source_commcare.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py index a6f3a7bc3a6f..6ee0e04d4463 100644 --- a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py @@ -9,12 +9,13 @@ from urllib.parse import parse_qs import requests +from flatten_json import flatten + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from flatten_json import flatten # Basic full refresh stream @@ -56,7 +57,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json"} return params @@ -79,7 +79,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json", "extras": "true"} return params @@ -123,7 +122,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json"} if next_page_token: params.update(next_page_token) @@ -136,7 +134,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Case(IncrementalStream): - """ docs: https://www.commcarehq.org/a/[domain]/api/[version]/case/ """ @@ -167,7 +164,6 @@ def path( def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - # start date is what we saved for forms # if self.cursor_field in self.state else (CommcareStream.last_form_date or self.initial_date) ix = self.state[self.cursor_field] @@ -234,7 +230,6 @@ def path( def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - # if self.cursor_field in self.state else self.initial_date ix = self.state[self.cursor_field] params = { diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commercetools/main.py b/airbyte-integrations/connectors/source-commercetools/main.py index 44dd2fb8f952..5d986dfc5e60 100644 --- a/airbyte-integrations/connectors/source-commercetools/main.py +++ b/airbyte-integrations/connectors/source-commercetools/main.py @@ -4,5 +4,6 @@ from source_commercetools.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py index f39ab7dd6f28..403963f6d594 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py @@ -7,9 +7,11 @@ import backoff import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py index 0d264787f978..0b5f63721a93 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_commercetools import SourceCommercetools +from airbyte_cdk.entrypoint import launch + def run(): source = SourceCommercetools() diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py index cdcca15f5522..b36bc81e6d50 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convex/main.py b/airbyte-integrations/connectors/source-convex/main.py index 751ae667fae2..c407ac73f395 100644 --- a/airbyte-integrations/connectors/source-convex/main.py +++ b/airbyte-integrations/connectors/source-convex/main.py @@ -4,5 +4,6 @@ from source_convex.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-convex/source_convex/source.py b/airbyte-integrations/connectors/source-convex/source_convex/source.py index 664f5bf3ca16..3b12ca62656f 100644 --- a/airbyte-integrations/connectors/source-convex/source_convex/source.py +++ b/airbyte-integrations/connectors/source-convex/source_convex/source.py @@ -8,12 +8,14 @@ from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, TypedDict, cast import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator + ConvexConfig = TypedDict( "ConvexConfig", { diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py index c1006b9f6167..a8416bcaca7e 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py @@ -5,10 +5,11 @@ from unittest.mock import MagicMock -from airbyte_cdk.models import SyncMode from pytest import fixture from source_convex.source import ConvexStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index 17512d01cf07..7fc5a7a0158f 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -8,9 +8,10 @@ import pytest import requests import responses -from airbyte_cdk.models import SyncMode from source_convex.source import ConvexStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def patch_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-declarative-manifest/main.py b/airbyte-integrations/connectors/source-declarative-manifest/main.py index fb1e853213d7..a4c78be28f71 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/main.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/main.py @@ -4,5 +4,6 @@ from source_declarative_manifest.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py b/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py index 2dfd888804b8..811567aa78fa 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any, List, Mapping, Optional +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import ( AirbyteErrorTraceMessage, @@ -27,7 +29,6 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import ConcurrentDeclarativeSource from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState -from orjson import orjson class SourceLocalYaml(YamlDeclarativeSource): diff --git a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py index e2f03e369f8d..41e6254d51f1 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py @@ -9,6 +9,7 @@ from jsonschema import ValidationError from source_declarative_manifest import run + POKEAPI_JSON_SPEC_SUBSTRING = '"required":["pokemon_name"]' SUCCESS_CHECK_SUBSTRING = '"connectionStatus":{"status":"SUCCEEDED"}' FAILED_CHECK_SUBSTRING = '"connectionStatus":{"status":"FAILED"}' @@ -16,8 +17,10 @@ @pytest.fixture(autouse=True) def setup(valid_local_manifest_yaml): - with patch('source_declarative_manifest.run._is_local_manifest_command', return_value=True): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=valid_local_manifest_yaml): + with patch("source_declarative_manifest.run._is_local_manifest_command", return_value=True): + with patch( + "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=valid_local_manifest_yaml + ): yield @@ -28,9 +31,9 @@ def test_spec_is_poke_api(capsys): def test_invalid_yaml_throws(capsys, invalid_local_manifest_yaml): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=invalid_local_manifest_yaml): - with pytest.raises(ValidationError): - run.handle_command(["spec"]) + with patch("source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=invalid_local_manifest_yaml): + with pytest.raises(ValidationError): + run.handle_command(["spec"]) def test_given_invalid_config_then_unsuccessful_check(capsys, invalid_local_config_file): diff --git a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py index 52a363a298c1..d96beb954d50 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py @@ -3,9 +3,11 @@ # import pytest -from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from source_declarative_manifest.run import create_declarative_source, handle_command +from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource + + REMOTE_MANIFEST_SPEC_SUBSTRING = '"required":["__injected_declarative_manifest"]' diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py index 3a0f562732fb..6e0d32803f45 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/main.py b/airbyte-integrations/connectors/source-facebook-marketing/main.py index fc25c7149e93..b0d0f19d8d9f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/main.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/main.py @@ -5,5 +5,6 @@ from source_facebook_marketing.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index 31bf4644013d..7b7615ea92b2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -13,8 +13,10 @@ from facebook_business.adobjects.adaccount import AdAccount from facebook_business.api import FacebookResponse from facebook_business.exceptions import FacebookRequestError + from source_facebook_marketing.streams.common import retry_pattern + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index da5b8dc1c285..a4b28821bce2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index b6eb3cb30c1e..776bdcbe2d67 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -7,6 +7,7 @@ import facebook_business import pendulum + from airbyte_cdk.models import ( AdvancedAuth, AuthFlowType, @@ -55,6 +56,7 @@ from .utils import validate_end_date, validate_start_date + logger = logging.getLogger("airbyte") UNSUPPORTED_FIELDS = {"unique_conversions", "unique_ctr", "unique_clicks"} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 79c82c2210ad..6ccf6edbf8b8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -6,14 +6,16 @@ from enum import Enum from typing import List, Literal, Optional, Set, Union -from airbyte_cdk.sources.config import BaseConfig -from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from facebook_business.adobjects.ad import Ad from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.adsinsights import AdsInsights from facebook_business.adobjects.campaign import Campaign from pydantic.v1 import BaseModel, Field, PositiveInt, constr +from airbyte_cdk.sources.config import BaseConfig +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig + + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index e8f1038c2bb9..c6da3d872767 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -18,10 +18,12 @@ from facebook_business.adobjects.objectparser import ObjectParser from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse from pendulum.duration import Duration + from source_facebook_marketing.streams.common import retry_pattern from ..utils import validate_start_date + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index dc01cd228412..4fcaddcfdfcc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -10,6 +10,7 @@ from .async_job import AsyncJob, ParentAsyncJob, update_in_batch + if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index b33b9278eeb7..8b85af62a4a3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -6,19 +6,21 @@ from functools import cache, cached_property from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union -import airbyte_cdk.sources.utils.casing as casing import pendulum +from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError + +import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.utils import AirbyteTracedException -from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob from source_facebook_marketing.streams.async_job_manager import InsightAsyncJobManager from source_facebook_marketing.streams.common import traced_exception from .base_streams import FBMarketingIncrementalStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index ed4bc29aef34..5e7562d644e9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -9,16 +9,18 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum +from facebook_business.adobjects.abstractobject import AbstractObject +from facebook_business.exceptions import FacebookRequestError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from facebook_business.adobjects.abstractobject import AbstractObject -from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams.common import traced_exception from .common import deep_merge + if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index c3446cdfea01..46f150e766f7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -10,9 +10,11 @@ import backoff import pendulum +from facebook_business.exceptions import FacebookRequestError + from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException -from facebook_business.exceptions import FacebookRequestError + # The Facebook API error codes indicating rate-limiting are listed at # https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index d33e202a637b..0dd3b4177a18 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -8,16 +8,18 @@ import pendulum import requests -from airbyte_cdk.models import SyncMode from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount from facebook_business.adobjects.adimage import AdImage from facebook_business.adobjects.user import User from facebook_business.exceptions import FacebookRequestError + +from airbyte_cdk.models import SyncMode from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses from .base_insight_streams import AdsInsights from .base_streams import FBMarketingIncrementalStream, FBMarketingReversedIncrementalStream, FBMarketingStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py index e81c6bfd14c5..71e35ddac007 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py @@ -8,6 +8,7 @@ import pendulum from pendulum import Date, DateTime + logger = logging.getLogger("airbyte") # Facebook store metrics maximum of 37 months old. Any time range that diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index 7a7cbaa39b9e..1ea8602ce656 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -7,6 +7,7 @@ from pytest import fixture from source_facebook_marketing.api import API + FB_API_VERSION = FacebookAdsApi.API_VERSION diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index 1e76b6403a50..51772039a52e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + ACCESS_TOKEN = "test_access_token" ACCOUNT_ID = "111111111111111" CLIENT_ID = "test_client_id" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py index 69b284d6d308..16846f6d78d4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py @@ -9,6 +9,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "QVFIUlhOX3Rnbm5Y" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 2a290cd48cbc..8bdbe9ddd9cc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -11,6 +11,8 @@ import freezegun import pendulum +from source_facebook_marketing.streams.async_job import Status + from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, StreamDescriptor, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker @@ -23,7 +25,6 @@ create_response_builder, find_template, ) -from source_facebook_marketing.streams.async_job import Status from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy @@ -31,113 +32,114 @@ from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response from .utils import config, encode_request_body, read_output + _STREAM_NAME = "ads_insights_action_product_id" _CURSOR_FIELD = "date_start" _REPORT_RUN_ID = "1571860060019548" _JOB_ID = "1049937379601625" _JOB_START_FIELDS = [ - "account_currency", - "account_id", - "account_name", - "action_values", - "actions", - "ad_click_actions", - "ad_id", - "ad_impression_actions", - "ad_name", - "adset_id", - "adset_name", - "attribution_setting", - "auction_bid", - "auction_competitiveness", - "auction_max_competitor_bid", - "buying_type", - "campaign_id", - "campaign_name", - "canvas_avg_view_percent", - "canvas_avg_view_time", - "catalog_segment_actions", - "catalog_segment_value", - "catalog_segment_value_mobile_purchase_roas", - "catalog_segment_value_omni_purchase_roas", - "catalog_segment_value_website_purchase_roas", - "clicks", - "conversion_rate_ranking", - "conversion_values", - "conversions", - "converted_product_quantity", - "converted_product_value", - "cost_per_15_sec_video_view", - "cost_per_2_sec_continuous_video_view", - "cost_per_action_type", - "cost_per_ad_click", - "cost_per_conversion", - "cost_per_estimated_ad_recallers", - "cost_per_inline_link_click", - "cost_per_inline_post_engagement", - "cost_per_outbound_click", - "cost_per_thruplay", - "cost_per_unique_action_type", - "cost_per_unique_click", - "cost_per_unique_inline_link_click", - "cost_per_unique_outbound_click", - "cpc", - "cpm", - "cpp", - "created_time", - "ctr", - "date_start", - "date_stop", - "engagement_rate_ranking", - "estimated_ad_recallers", - "frequency", - "full_view_impressions", - "full_view_reach", - "impressions", - "inline_link_click_ctr", - "inline_link_clicks", - "inline_post_engagement", - "instant_experience_clicks_to_open", - "instant_experience_clicks_to_start", - "instant_experience_outbound_clicks", - "mobile_app_purchase_roas", - "objective", - "optimization_goal", - "outbound_clicks", - "outbound_clicks_ctr", - "purchase_roas", - "qualifying_question_qualify_answer_rate", - "quality_ranking", - "reach", - "social_spend", - "spend", - "unique_actions", - "unique_clicks", - "unique_ctr", - "unique_inline_link_click_ctr", - "unique_inline_link_clicks", - "unique_link_clicks_ctr", - "unique_outbound_clicks", - "unique_outbound_clicks_ctr", - "updated_time", - "video_15_sec_watched_actions", - "video_30_sec_watched_actions", - "video_avg_time_watched_actions", - "video_continuous_2_sec_watched_actions", - "video_p100_watched_actions", - "video_p25_watched_actions", - "video_p50_watched_actions", - "video_p75_watched_actions", - "video_p95_watched_actions", - "video_play_actions", - "video_play_curve_actions", - "video_play_retention_0_to_15s_actions", - "video_play_retention_20_to_60s_actions", - "video_play_retention_graph_actions", - "video_time_watched_actions", - "website_ctr", - "website_purchase_roas", - ] + "account_currency", + "account_id", + "account_name", + "action_values", + "actions", + "ad_click_actions", + "ad_id", + "ad_impression_actions", + "ad_name", + "adset_id", + "adset_name", + "attribution_setting", + "auction_bid", + "auction_competitiveness", + "auction_max_competitor_bid", + "buying_type", + "campaign_id", + "campaign_name", + "canvas_avg_view_percent", + "canvas_avg_view_time", + "catalog_segment_actions", + "catalog_segment_value", + "catalog_segment_value_mobile_purchase_roas", + "catalog_segment_value_omni_purchase_roas", + "catalog_segment_value_website_purchase_roas", + "clicks", + "conversion_rate_ranking", + "conversion_values", + "conversions", + "converted_product_quantity", + "converted_product_value", + "cost_per_15_sec_video_view", + "cost_per_2_sec_continuous_video_view", + "cost_per_action_type", + "cost_per_ad_click", + "cost_per_conversion", + "cost_per_estimated_ad_recallers", + "cost_per_inline_link_click", + "cost_per_inline_post_engagement", + "cost_per_outbound_click", + "cost_per_thruplay", + "cost_per_unique_action_type", + "cost_per_unique_click", + "cost_per_unique_inline_link_click", + "cost_per_unique_outbound_click", + "cpc", + "cpm", + "cpp", + "created_time", + "ctr", + "date_start", + "date_stop", + "engagement_rate_ranking", + "estimated_ad_recallers", + "frequency", + "full_view_impressions", + "full_view_reach", + "impressions", + "inline_link_click_ctr", + "inline_link_clicks", + "inline_post_engagement", + "instant_experience_clicks_to_open", + "instant_experience_clicks_to_start", + "instant_experience_outbound_clicks", + "mobile_app_purchase_roas", + "objective", + "optimization_goal", + "outbound_clicks", + "outbound_clicks_ctr", + "purchase_roas", + "qualifying_question_qualify_answer_rate", + "quality_ranking", + "reach", + "social_spend", + "spend", + "unique_actions", + "unique_clicks", + "unique_ctr", + "unique_inline_link_click_ctr", + "unique_inline_link_clicks", + "unique_link_clicks_ctr", + "unique_outbound_clicks", + "unique_outbound_clicks_ctr", + "updated_time", + "video_15_sec_watched_actions", + "video_30_sec_watched_actions", + "video_avg_time_watched_actions", + "video_continuous_2_sec_watched_actions", + "video_p100_watched_actions", + "video_p25_watched_actions", + "video_p50_watched_actions", + "video_p75_watched_actions", + "video_p95_watched_actions", + "video_play_actions", + "video_play_curve_actions", + "video_play_retention_0_to_15s_actions", + "video_play_retention_20_to_60s_actions", + "video_play_retention_graph_actions", + "video_time_watched_actions", + "website_ctr", + "website_purchase_roas", +] def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: @@ -145,7 +147,10 @@ def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) - def _job_start_request( - account_id: Optional[str] = ACCOUNT_ID, since: Optional[datetime] = None, until: Optional[datetime] = None, fields: Optional[List[str]] = None + account_id: Optional[str] = ACCOUNT_ID, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + fields: Optional[List[str]] = None, ) -> RequestBuilder: since = since.strftime(DATE_FORMAT) if since else START_DATE[:10] until = until.strftime(DATE_FORMAT) if until else END_DATE[:10] @@ -174,7 +179,7 @@ def _job_start_request( "PENDING_BILLING_INFO", "PENDING_REVIEW", "PREAPPROVED", - "WITH_ISSUES" + "WITH_ISSUES", ], }, ], @@ -250,7 +255,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -434,7 +439,10 @@ def test_given_status_500_reduce_amount_of_data_when_read_then_limit_reduced(sel class TestIncremental(TestCase): @staticmethod def _read( - config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False, json_schema: Optional[Dict[str, any]] = None + config_: ConfigBuilder, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: bool = False, + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: return read_output( config_builder=config_, @@ -442,7 +450,7 @@ def _read( sync_mode=SyncMode.incremental, state=state, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -467,7 +475,9 @@ def test_when_read_then_state_message_produced_and_state_match_start_interval(se ) output = self._read(config().with_account_ids([account_id]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_message == start_date.strftime(DATE_FORMAT) @@ -511,8 +521,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) expected_cursor_value = start_date.strftime(DATE_FORMAT) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_account_1 == expected_cursor_value diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index d437566f1d48..90c13e2a6941 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -7,6 +7,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker @@ -26,6 +27,7 @@ from .response_builder import error_reduce_amount_of_data_response, get_account_response from .utils import config, read_output + _STREAM_NAME = "videos" _CURSOR_FIELD = "updated_time" _FIELDS = [ @@ -96,7 +98,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -244,7 +246,9 @@ def test_when_read_then_state_message_produced_and_state_match_latest_record(sel ) output = self._read(config().with_account_ids([account_id])) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_message == max_cursor_value @HttpMocker() @@ -276,8 +280,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2])) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_account_1 == max_cursor_value_account_id_1 assert cursor_value_from_state_account_2 == max_cursor_value_account_id_2 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py index 2686186e1840..eb45b856b124 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py @@ -6,11 +6,12 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlencode +from facebook_business.api import _top_level_param_json_encode +from source_facebook_marketing import SourceFacebookMarketing + from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import ConfiguredAirbyteStreamBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from facebook_business.api import _top_level_param_json_encode -from source_facebook_marketing import SourceFacebookMarketing from .config import ConfigBuilder @@ -34,7 +35,7 @@ def read_output( sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: Optional[bool] = False, - json_schema: Optional[Dict[str, any]] = None + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: _catalog = catalog(stream_name, sync_mode, json_schema) _config = config_builder.build() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py index c09279ca1d8d..0bd249e56bc4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py @@ -8,6 +8,7 @@ from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.adobjects.adaccount import AdAccount + FB_API_VERSION = FacebookAdsApi.API_VERSION diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 9b9a0e2a1f9f..86a58ffadcfa 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -6,12 +6,13 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from freezegun import freeze_time from pendulum import duration from source_facebook_marketing.streams import AdsInsights from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob +from airbyte_cdk.models import SyncMode + @pytest.fixture(name="api") def api_fixture(mocker): @@ -99,14 +100,10 @@ def test_init_statuses(self, api, some_config): end_date=datetime(2011, 1, 1), insights_lookback_window=28, fields=["account_id", "account_currency"], - filter_statuses=["ACTIVE", "ARCHIVED"] + filter_statuses=["ACTIVE", "ARCHIVED"], ) - assert stream.request_params()["filtering"] == [ - {'field': 'ad.effective_status', - 'operator': 'IN', - 'value': ['ACTIVE', 'ARCHIVED']} - ] + assert stream.request_params()["filtering"] == [{"field": "ad.effective_status", "operator": "IN", "value": ["ACTIVE", "ARCHIVED"]}] def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock @@ -464,7 +461,9 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) - assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1, "should be 37 slices because we ignore slices which are within insights_lookback_period" + assert ( + len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1 + ), "should be 37 slices because we ignore slices which are within insights_lookback_period" assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1) @@ -602,61 +601,78 @@ def test_start_date_with_lookback_window( @pytest.mark.parametrize( "breakdowns, record, expected_record", ( - ( - ["body_asset", ], - {"body_asset": {"id": "871246182", "text": "Some text"}}, - {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"} - ), - ( - ["call_to_action_asset",], - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"} - ), - ( - ["description_asset", ], - {"description_asset": {"id": "871246182", "text": "Some text"}}, - {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"} - ), - ( - ["image_asset", ], - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"} - ), - ( - ["link_url_asset", ], - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"} - ), - ( - ["title_asset", ], - {"title_asset": {"id": "871246182", "text": "Some text"}}, - {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"} - ), - ( - ["video_asset", ], - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - } - }, - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - }, - "video_asset_id": "871246182" - } - ), - ( - ["body_asset", "country"], - {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, - { - "body_asset": {"id": "871246182", "text": "Some text"}, - "country": "country", "dma": "dma", "body_asset_id": "871246182" - } - ), - ) + ( + [ + "body_asset", + ], + {"body_asset": {"id": "871246182", "text": "Some text"}}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"}, + ), + ( + [ + "call_to_action_asset", + ], + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"}, + ), + ( + [ + "description_asset", + ], + {"description_asset": {"id": "871246182", "text": "Some text"}}, + {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"}, + ), + ( + [ + "image_asset", + ], + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"}, + ), + ( + [ + "link_url_asset", + ], + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"}, + ), + ( + [ + "title_asset", + ], + {"title_asset": {"id": "871246182", "text": "Some text"}}, + {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"}, + ), + ( + [ + "video_asset", + ], + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + } + }, + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + }, + "video_asset_id": "871246182", + }, + ), + ( + ["body_asset", "country"], + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma", "body_asset_id": "871246182"}, + ), + ), ) def test_transform_breakdowns(self, api, some_config, breakdowns, record, expected_record): start_date = pendulum.parse("2024-01-01") @@ -674,36 +690,19 @@ def test_transform_breakdowns(self, api, some_config, breakdowns, record, expect @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - None, - ["date_start", "account_id", "ad_id"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + (None, ["date_start", "account_id", "ad_id"]), + ), ) def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -714,43 +713,38 @@ def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) assert stream.primary_key == expect_pks @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], - ["date_start", "account_id", "ad_id", "video_asset_id", "link_url_asset_id", "skan_conversion_id", "place_page_id", "gender"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + ( + ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], + [ + "date_start", + "account_id", + "ad_id", + "video_asset_id", + "link_url_asset_id", + "skan_conversion_id", + "place_page_id", + "gender", + ], + ), + ), ) def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -761,7 +755,7 @@ def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pk start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) schema = stream.get_json_schema() assert schema diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 885560c92d3a..65b53d21eb11 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -6,12 +6,14 @@ import pendulum import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams import Activities, AdAccount, AdCreatives, Campaigns, Videos +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException + + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -105,7 +107,9 @@ def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, except FacebookRequestError: pytest.fail("Call rate error has not being handled") - def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception(self, requests_mock, api, fb_call_rate_response, account_id, some_config): + def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception( + self, requests_mock, api, fb_call_rate_response, account_id, some_config + ): requests_mock.register_uri( "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 68229d5923a1..91a1a9b1a174 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -8,8 +8,6 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_facebook_marketing.config_migrations import ( MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters, @@ -17,6 +15,10 @@ ) from source_facebook_marketing.source import SourceFacebookMarketing +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" SOURCE: Source = SourceFacebookMarketing() @@ -168,6 +170,7 @@ def test_should_not_migrate_upgraded_config(self): migration_instance = MigrateIncludeDeletedToStatusFilters() assert not migration_instance.should_migrate(new_config) + class TestMigrateSecretsPathInConnector: OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_access_token_config.json") NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_access_token_config.json") @@ -178,7 +181,7 @@ class TestMigrateSecretsPathInConnector: def revert_migration(config_path: str) -> None: with open(config_path, "r") as test_config: config = json.load(test_config) - credentials = config.pop("credentials",{}) + credentials = config.pop("credentials", {}) credentials.pop("auth_type", None) with open(config_path, "w") as updated_config: config = json.dumps({**config, **credentials}) @@ -202,7 +205,7 @@ def test_migrate_access_token_config(self): assert original_config["access_token"] == credentials["access_token"] # revert the test_config to the starting point self.revert_migration(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) - + def test_migrate_client_config(self): migration_instance = MigrateSecretsPathInConnector() original_config = load_config(self.OLD_TEST_CONFIG_PATH_CLIENT) @@ -228,7 +231,7 @@ def test_should_not_migrate_new_client_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) migration_instance = MigrateSecretsPathInConnector() assert not migration_instance._should_migrate(new_config) - + def test_should_not_migrate_new_access_token_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_ACCESS_TOKEN) migration_instance = MigrateSecretsPathInConnector() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py index b593e7e1b1f3..258c32425638 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -7,14 +7,16 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.api import API from source_facebook_marketing.streams import AdAccount, AdCreatives, AdsInsights from source_facebook_marketing.streams.common import traced_exception +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + FB_API_VERSION = FacebookAdsApi.API_VERSION account_id = "unknown_account" @@ -113,7 +115,7 @@ "code": 100, } }, - } + }, # Error randomly happens for different connections. # Can be reproduced on https://developers.facebook.com/tools/explorer/?method=GET&path=act_&version=v17.0 # 1st reason: incorrect ad account id is used @@ -183,7 +185,7 @@ "error_user_msg": "profile should always be linked to delegate page", } }, - } + }, # Error happens on Video stream: https://graph.facebook.com/v17.0/act_XXXXXXXXXXXXXXXX/advideos # Recommendations says that the problem can be fixed by switching to Business Ad Account Id ), @@ -215,8 +217,8 @@ "message": "(#3018) The start date of the time range cannot be beyond 37 months from the current date", "type": "OAuthException", "code": 3018, - "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P" - } + "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P", + } }, }, ), @@ -252,27 +254,28 @@ SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME = "error_400_service_temporarily_unavailable" SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE = { - "status_code": 503, - "json": { - "error": { - "message": "(#2) Service temporarily unavailable", - "type": "OAuthException", - "is_transient": True, - "code": 2, - "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", - } - }, - } + "status_code": 503, + "json": { + "error": { + "message": "(#2) Service temporarily unavailable", + "type": "OAuthException", + "is_transient": True, + "code": 2, + "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", + } + }, +} REDUCE_FIELDS_ERROR_TEST_NAME = "error_500_reduce_the_amount_of_data" REDUCE_FIELDS_ERROR_RESPONSE = { - "status_code": 500, - "json": { - "error": { - "message": "Please reduce the amount of data you're asking for, then retry your request", - "code": 1, - } - }, - } + "status_code": 500, + "json": { + "error": { + "message": "Please reduce the amount of data you're asking for, then retry your request", + "code": 1, + } + }, +} + class TestRealErrors: @pytest.mark.parametrize( @@ -406,23 +409,24 @@ def test_config_error_during_actual_nodes_read(self, requests_mock, name, friend assert error.failure_type == FailureType.config_error assert friendly_msg in error.message - @pytest.mark.parametrize("name, friendly_msg, config_error_response, failure_type", - [ - ( - REDUCE_FIELDS_ERROR_TEST_NAME, - "Please reduce the number of fields requested. Go to the schema tab, " - "select your source, and unselect the fields you do not need.", - REDUCE_FIELDS_ERROR_RESPONSE, - FailureType.config_error - ), - ( - SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, - "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", - SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, - FailureType.transient_error - ) - ] - ) + @pytest.mark.parametrize( + "name, friendly_msg, config_error_response, failure_type", + [ + ( + REDUCE_FIELDS_ERROR_TEST_NAME, + "Please reduce the number of fields requested. Go to the schema tab, " + "select your source, and unselect the fields you do not need.", + REDUCE_FIELDS_ERROR_RESPONSE, + FailureType.config_error, + ), + ( + SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, + "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", + SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, + FailureType.transient_error, + ), + ], + ) def test_config_error_that_was_retried_when_reading_nodes(self, requests_mock, name, friendly_msg, config_error_response, failure_type): """This test covers errors that have been resolved in the past with a retry strategy, but it could also can fail after retries, then, we need to provide the user with a humanized error explaining what just happened""" @@ -498,7 +502,7 @@ def test_config_error_insights_during_actual_nodes_read(self, requests_mock, nam assert friendly_msg in error.message def test_retry_for_cannot_include_error(self, requests_mock): - """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868 """ + """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868""" api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( @@ -516,7 +520,7 @@ def test_retry_for_cannot_include_error(self, requests_mock): "error": { "message": "(#100) Cannot include video_avg_time_watched_actions, video_continuous_2_sec_watched_actions in summary param because they weren't there while creating the report run.", "type": "OAuthException", - "code": 100 + "code": 100, } }, } @@ -594,24 +598,22 @@ def test_traced_exception_with_api_error(): request_context={}, http_status=400, http_headers={}, - body='{"error": {"message": "Error validating access token", "code": 190}}' + body='{"error": {"message": "Error validating access token", "code": 190}}', ) error.api_error_message = MagicMock(return_value="Error validating access token") - + result = traced_exception(error) - + assert isinstance(result, AirbyteTracedException) - assert result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + assert ( + result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + ) assert result.failure_type == FailureType.config_error def test_traced_exception_without_api_error(): error = FacebookRequestError( - message="Call was unsuccessful. The Facebook API has imploded", - request_context={}, - http_status=408, - http_headers={}, - body='{}' + message="Call was unsuccessful. The Facebook API has imploded", request_context={}, http_status=408, http_headers={}, body="{}" ) error.api_error_message = MagicMock(return_value=None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 774f83c1c9e5..c5a654daa7c9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -7,6 +7,10 @@ from unittest.mock import call import pytest +from facebook_business import FacebookAdsApi, FacebookSession +from source_facebook_marketing import SourceFacebookMarketing +from source_facebook_marketing.spec import ConnectorConfig + from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -18,9 +22,6 @@ Status, SyncMode, ) -from facebook_business import FacebookAdsApi, FacebookSession -from source_facebook_marketing import SourceFacebookMarketing -from source_facebook_marketing.spec import ConnectorConfig from .utils import command_check diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py index 3a0ac0691c2c..209870c38ccd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py @@ -7,6 +7,7 @@ import pytest from source_facebook_marketing.utils import DATA_RETENTION_PERIOD, validate_end_date, validate_start_date + TODAY = pendulum.datetime(2023, 3, 31) diff --git a/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-pages/main.py b/airbyte-integrations/connectors/source-facebook-pages/main.py index 466fc2800442..ed626c5ef709 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/main.py +++ b/airbyte-integrations/connectors/source-facebook-pages/main.py @@ -4,5 +4,6 @@ from source_facebook_pages.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py index d730dc2ba169..07df178e89d5 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py @@ -9,12 +9,13 @@ import dpath.util import pendulum import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState -from requests import HTTPError @dataclass diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py index 3b70710fe59d..9dd391453971 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_facebook_pages import SourceFacebookPages +from airbyte_cdk.entrypoint import launch + def run(): source = SourceFacebookPages() diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py index c7f4b7e08d91..e6b19d27cc0c 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py b/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py index 97e14aa22c05..ae98e528c865 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py +++ b/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py @@ -107,6 +107,7 @@ import json import os + spec_path = "facebook-business-sdk-codegen/api_specs/specs" fb_node_files = os.listdir(spec_path) fb_node_files.sort() @@ -230,7 +231,6 @@ def is_node(name): def get_fields(fields, with_refs=False): - # process Node's fields schema_fields = {} for attr in fields: @@ -282,7 +282,6 @@ def get_fields(fields, with_refs=False): def get_edges(edges): - schema_edges = {} attrs = {} for attr in edges: @@ -329,7 +328,6 @@ def get_edges(edges): def build_schema(node_name, with_refs=False): - file_path = f"{spec_path}/{node_name}.json" print(f"Fetching schema from file: {file_path}") @@ -353,7 +351,6 @@ def build_schema(node_name, with_refs=False): print(f"Process main nodes: {MAIN_NODES}") for node_name in MAIN_NODES: - page_schema = build_schema(node_name=node_name, with_refs=True) SCHEMA = {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": page_schema} diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py index 36fe896ca5d8..8f553d3d745c 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py @@ -9,9 +9,10 @@ def test_field_transformation(): - with open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, open( - f"{os.path.dirname(__file__)}/transformed_record.json", "r" - ) as transformed_record: + with ( + open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, + open(f"{os.path.dirname(__file__)}/transformed_record.json", "r") as transformed_record, + ): initial_record = json.loads(initial_record.read()) transformed_record = json.loads(transformed_record.read()) record_transformation = CustomFieldTransformation(config={}, parameters={"name": "page"}) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py index 9df2974ae7bd..27ee46fc769b 100644 --- a/airbyte-integrations/connectors/source-faker/main.py +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -5,5 +5,6 @@ from source_faker.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py index 70f166a8a4fb..6be3cd5510c5 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -6,9 +6,10 @@ from multiprocessing import current_process from typing import Dict, List -from airbyte_cdk.models import AirbyteRecordMessage, Type from mimesis import Datetime, Numeric +from airbyte_cdk.models import AirbyteRecordMessage, Type + from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON from .utils import format_airbyte_time, now_millis diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index 15423b7fcb81..965d2b71b68a 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -10,6 +10,7 @@ from .streams import Products, Purchases, Users + DEFAULT_COUNT = 1_000 diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py index 2e8a0b7b2192..4251b42ac01b 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -5,10 +5,11 @@ import datetime from multiprocessing import current_process -from airbyte_cdk.models import AirbyteRecordMessage, Type from mimesis import Address, Datetime, Person from mimesis.locales import Locale +from airbyte_cdk.models import AirbyteRecordMessage, Type + from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON from .utils import format_airbyte_time, now_millis diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index e4fbef60201b..395a0181e47b 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -4,9 +4,10 @@ import jsonschema import pytest -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type from source_faker import SourceFaker +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type + class MockLogger: def debug(a, b, **kwargs): diff --git a/airbyte-integrations/connectors/source-fastbill/components.py b/airbyte-integrations/connectors/source-fastbill/components.py index 002f96dd713c..2d3160cd6992 100644 --- a/airbyte-integrations/connectors/source-fastbill/components.py +++ b/airbyte-integrations/connectors/source-fastbill/components.py @@ -12,7 +12,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self): - username = self._username.eval(self.config).encode("latin1") password = self._password.eval(self.config).encode("latin1") encoded_credentials = base64.b64encode(b":".join((username, password))).strip() diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fauna/main.py b/airbyte-integrations/connectors/source-fauna/main.py index 9e4bc25307ed..85e9d886d239 100644 --- a/airbyte-integrations/connectors/source-fauna/main.py +++ b/airbyte-integrations/connectors/source-fauna/main.py @@ -4,5 +4,6 @@ from source_fauna.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index 25c4e60b8647..83c3a7c2ea4d 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "faunadb~=4.2", diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py index 8cc685a1586c..eb1d5cdbc9f2 100644 --- a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py @@ -8,6 +8,12 @@ from datetime import datetime from typing import Dict, Generator, Optional +from faunadb import _json +from faunadb import query as q +from faunadb.client import FaunaClient +from faunadb.errors import FaunaError, Unauthorized +from faunadb.objects import Ref + from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ( AirbyteCatalog, @@ -23,11 +29,6 @@ Type, ) from airbyte_cdk.sources import Source -from faunadb import _json -from faunadb import query as q -from faunadb.client import FaunaClient -from faunadb.errors import FaunaError, Unauthorized -from faunadb.objects import Ref from source_fauna.serialize import fauna_doc_to_airbyte diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py index c6c8263cc852..5c6453b7748d 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py @@ -4,13 +4,14 @@ from unittest.mock import MagicMock, Mock -from airbyte_cdk.models import Status from faunadb import query as q from faunadb.errors import Unauthorized from faunadb.objects import Ref from source_fauna import SourceFauna from test_util import config, mock_logger +from airbyte_cdk.models import Status + def query_hardcoded(expr): print(expr) diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py index 5137d192adaf..1629a6bc530f 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py @@ -11,6 +11,10 @@ from datetime import datetime import docker +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, DeletionsConfig, FullConfig, config, mock_logger, ref + from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteStream, @@ -21,9 +25,6 @@ SyncMode, Type, ) -from faunadb import query as q -from source_fauna import SourceFauna -from test_util import CollectionConfig, DeletionsConfig, FullConfig, config, mock_logger, ref def setup_database(source: SourceFauna): diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py index 82ac8183af34..7def4507c2ff 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py @@ -4,12 +4,13 @@ from unittest.mock import MagicMock, Mock -from airbyte_cdk.models import AirbyteStream from faunadb import query as q from faunadb.objects import Ref from source_fauna import SourceFauna from test_util import config, mock_logger +from airbyte_cdk.models import AirbyteStream + def mock_source() -> SourceFauna: source = SourceFauna() diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py index 9a955244a5d4..a95880aad353 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py @@ -6,6 +6,11 @@ from typing import Dict, Generator from unittest.mock import MagicMock, Mock +from faunadb import _json +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, config, expand_columns_query, mock_logger, ref + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,10 +22,7 @@ SyncMode, Type, ) -from faunadb import _json -from faunadb import query as q -from source_fauna import SourceFauna -from test_util import CollectionConfig, config, expand_columns_query, mock_logger, ref + NOW = 1234512987 diff --git a/airbyte-integrations/connectors/source-file/build_customization.py b/airbyte-integrations/connectors/source-file/build_customization.py index 1c585de9caac..83411796d153 100644 --- a/airbyte-integrations/connectors/source-file/build_customization.py +++ b/airbyte-integrations/connectors/source-file/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py index 758c1118eae6..38adaeb45a1f 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py @@ -9,6 +9,7 @@ import pytest from source_file.client import Client + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py index cb8041a4f396..465dcb84e2bf 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -24,6 +24,7 @@ from paramiko.client import AutoAddPolicy, SSHClient from paramiko.ssh_exception import SSHException + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py index 3f9980d1c0bf..69379aa6586b 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py @@ -7,10 +7,12 @@ from pathlib import Path import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_file import SourceFile from source_file.client import Client +from airbyte_cdk.utils import AirbyteTracedException + + SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/formats") diff --git a/airbyte-integrations/connectors/source-file/main.py b/airbyte-integrations/connectors/source-file/main.py index 3e7e82fd61d8..98e8e365ff72 100644 --- a/airbyte-integrations/connectors/source-file/main.py +++ b/airbyte-integrations/connectors/source-file/main.py @@ -4,5 +4,6 @@ from source_file.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index 2a0e021df1cb..ca197b53d3d0 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -24,9 +24,6 @@ import pandas as pd import smart_open import smart_open.ssh -from airbyte_cdk.entrypoint import logger -from airbyte_cdk.models import AirbyteStream, FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment from azure.storage.blob import BlobServiceClient from genson import SchemaBuilder from google.cloud.storage import Client as GCSClient @@ -38,8 +35,13 @@ from urllib3.exceptions import ProtocolError from yaml import safe_load +from airbyte_cdk.entrypoint import logger +from airbyte_cdk.models import AirbyteStream, FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment + from .utils import LOCAL_STORAGE_NAME, backoff_handler + SSH_TIMEOUT = 60 # Force the log level of the smart-open logger to ERROR - https://github.com/airbytehq/airbyte/pull/27157 diff --git a/airbyte-integrations/connectors/source-file/source_file/utils.py b/airbyte-integrations/connectors/source-file/source_file/utils.py index ef8c258b9179..248ee36e9f6e 100644 --- a/airbyte-integrations/connectors/source-file/source_file/utils.py +++ b/airbyte-integrations/connectors/source-file/source_file/utils.py @@ -5,6 +5,7 @@ import logging from urllib.parse import parse_qs, urlencode, urlparse + # default logger logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index fc28d7e66c55..326f6475326e 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -8,13 +8,14 @@ import pandas as pd import pytest -from airbyte_cdk.utils import AirbyteTracedException from pandas import read_csv, read_excel, testing from paramiko import SSHException from source_file.client import Client, URLFile from source_file.utils import backoff_handler from urllib3.exceptions import ProtocolError +from airbyte_cdk.utils import AirbyteTracedException + @pytest.fixture def wrong_format_client(): @@ -99,8 +100,7 @@ def test_load_dataframes_xlsx(config, absolute_path, test_files, file_name, shou assert read_file.equals(expected) -@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), - ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) +@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) def test_load_nested_json(client, config, absolute_path, test_files, file_format, file_path): if file_format == "jsonl": config["format"] = file_format @@ -131,7 +131,8 @@ def test_cache_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv" with open(f, mode="rb") as file: assert client._cache_stream(file) - + + def test_unzip_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv.zip" with open(f, mode="rb") as file: @@ -223,10 +224,11 @@ def patched_open(self): def test_backoff_handler(caplog): details = {"tries": 1, "wait": 1} backoff_handler(details) - expected = [('airbyte', 20, 'Caught retryable error after 1 tries. Waiting 1 seconds then retrying...')] + expected = [("airbyte", 20, "Caught retryable error after 1 tries. Waiting 1 seconds then retrying...")] assert caplog.record_tuples == expected + def generate_excel_file(data): """ Helper function to generate an Excel file with the given data. diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py b/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py index 14bd37a419aa..8466ad5c82f2 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py @@ -10,6 +10,7 @@ import pytest from source_file.source import SourceFile + json_obj = { "id": "0001", "type": "donut", diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py index 0a0c0ca798ad..26446e5db6d1 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py @@ -8,6 +8,8 @@ import jsonschema import pytest +from source_file.source import SourceFile + from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteMessage, @@ -22,7 +24,7 @@ ) from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models.airbyte_protocol import Type as MessageType -from source_file.source import SourceFile + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py index 50bba3cdce94..b7498c7869f7 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py @@ -10,6 +10,7 @@ import docker import pytest + pytest_plugins = ("source_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py index 708648fa8c15..58586ddc6928 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py @@ -4,5 +4,6 @@ from source_firebase_realtime_database.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py index a1370a44d289..4de83ba31dda 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py @@ -35,7 +35,6 @@ ], ) def test_stream_name_from(config, stream_name): - actual = SourceFirebaseRealtimeDatabase.stream_name_from(config) expected = stream_name diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py index c66074e0551b..d6c185fd5de8 100644 --- a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py @@ -8,6 +8,10 @@ from typing import Dict, Generator from unittest.mock import MagicMock +from firebolt.db import Connection +from pytest import fixture +from source_firebolt.source import SourceFirebolt, establish_connection + from airbyte_cdk.models import Status from airbyte_cdk.models.airbyte_protocol import ( AirbyteStream, @@ -16,9 +20,6 @@ DestinationSyncMode, SyncMode, ) -from firebolt.db import Connection -from pytest import fixture -from source_firebolt.source import SourceFirebolt, establish_connection @fixture(scope="module") diff --git a/airbyte-integrations/connectors/source-firebolt/main.py b/airbyte-integrations/connectors/source-firebolt/main.py index a901e9c4ae29..3aa31e6e1953 100644 --- a/airbyte-integrations/connectors/source-firebolt/main.py +++ b/airbyte-integrations/connectors/source-firebolt/main.py @@ -4,5 +4,6 @@ from source_firebolt.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py index 2d53f9c90663..aaf4e8112643 100644 --- a/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py @@ -20,6 +20,7 @@ from .database import establish_connection, get_table_structure from .utils import airbyte_message_from_data, convert_type + SUPPORTED_SYNC_MODES = [SyncMode.full_refresh] diff --git a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py index 2e273232b323..01ab8d494202 100644 --- a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py +++ b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py @@ -6,6 +6,11 @@ from decimal import Decimal from unittest.mock import MagicMock, patch +from pytest import fixture, mark +from source_firebolt.database import get_table_structure, parse_config +from source_firebolt.source import SUPPORTED_SYNC_MODES, SourceFirebolt, convert_type, establish_connection +from source_firebolt.utils import airbyte_message_from_data, format_fetch_result + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,10 +22,6 @@ SyncMode, Type, ) -from pytest import fixture, mark -from source_firebolt.database import get_table_structure, parse_config -from source_firebolt.source import SUPPORTED_SYNC_MODES, SourceFirebolt, convert_type, establish_connection -from source_firebolt.utils import airbyte_message_from_data, format_fetch_result @fixture(params=["my_engine", "my_engine.api.firebolt.io"]) @@ -33,6 +34,7 @@ def config(request): } return args + @fixture() def legacy_config(request): args = { @@ -44,6 +46,7 @@ def legacy_config(request): } return args + @fixture() def config_no_engine(): args = { @@ -109,12 +112,14 @@ def test_parse_config(config, logger): result = parse_config(config, logger) assert result["engine_url"] == "override_engine.api.firebolt.io" + def test_parse_legacy_config(legacy_config, logger): result = parse_config(legacy_config, logger) assert result["database"] == "my_database" assert result["auth"].username == "my@username" assert result["auth"].password == "my_password" + @patch("source_firebolt.database.connect") def test_connection(mock_connection, config, config_no_engine, logger): establish_connection(config, logger) diff --git a/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py index 7039ceb25a6d..c990d3691ce9 100644 --- a/airbyte-integrations/connectors/source-freshcaller/main.py +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -4,5 +4,6 @@ from source_freshcaller.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py index b6757d75d1aa..d4c01966794a 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_freshcaller import SourceFreshcaller +from airbyte_cdk.entrypoint import launch + def run(): source = SourceFreshcaller() diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py index 61eb6c017422..edfa285746a6 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshdesk/main.py b/airbyte-integrations/connectors/source-freshdesk/main.py index d32eaa6ca9e5..dc9d287ffe12 100644 --- a/airbyte-integrations/connectors/source-freshdesk/main.py +++ b/airbyte-integrations/connectors/source-freshdesk/main.py @@ -4,5 +4,6 @@ from source_freshdesk.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py index 3c87fa809e6a..62c441218c17 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement @@ -109,7 +110,9 @@ def get_request_params( start_time = stream_slice.get(self._partition_field_start.eval(self.config)) if not self.updated_slice else self.updated_slice options[self.start_time_option.field_name.eval(config=self.config)] = start_time # type: ignore # field_name is always casted to an interpolated string if self.end_time_option and self.end_time_option.inject_into == option_type: - options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(self._partition_field_end.eval(self.config)) # type: ignore # field_name is always casted to an interpolated string + options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get( + self._partition_field_end.eval(self.config) + ) # type: ignore # field_name is always casted to an interpolated string return options diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py index 982a82aa8163..533d78205ce1 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models import SyncMode from conftest import find_stream +from airbyte_cdk.models import SyncMode + @pytest.fixture(name="responses") def responses_fixtures(): diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py index 619ad510a506..cf598caf2629 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py @@ -2,9 +2,10 @@ import pytest -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from source_freshdesk.components import FreshdeskTicketsIncrementalSync +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType + class TestFreshdeskTicketsIncrementalSync: @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py index 4cdcf74fe6c1..e5ac47ab81b5 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py @@ -8,7 +8,6 @@ class TestFreshdeskTicketsPaginationStrategy: - # returns None when there are fewer records than the page size @pytest.mark.parametrize( "response, current_page, last_records, expected", diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index 1e5a1444574c..2ca8dc18a996 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -6,9 +6,10 @@ from typing import Any, MutableMapping import pytest +from conftest import find_stream + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream -from conftest import find_stream def _read_full_refresh(stream_instance: Stream): diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gcs/build_customization.py b/airbyte-integrations/connectors/source-gcs/build_customization.py index 83b1b136bf90..33593a539450 100644 --- a/airbyte-integrations/connectors/source-gcs/build_customization.py +++ b/airbyte-integrations/connectors/source-gcs/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py index 1fe90e1f1374..639577406d69 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py @@ -14,6 +14,7 @@ from .utils import get_docker_ip + LOCAL_GCP_PORT = 4443 from urllib.parse import urlparse, urlunparse @@ -77,4 +78,3 @@ def connector_setup_fixture(docker_client) -> None: container.kill() container.remove() - diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py index e14dd36501e1..7046c434ca28 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py @@ -2,10 +2,11 @@ import os import pytest +from source_gcs import Config, SourceGCS, SourceGCSStreamReader + from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import read -from source_gcs import Config, SourceGCS, SourceGCSStreamReader from .utils import load_config diff --git a/airbyte-integrations/connectors/source-gcs/main.py b/airbyte-integrations/connectors/source-gcs/main.py index 695a41f39be3..3b58be6e85f0 100644 --- a/airbyte-integrations/connectors/source-gcs/main.py +++ b/airbyte-integrations/connectors/source-gcs/main.py @@ -5,5 +5,6 @@ from source_gcs.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py index 40c5ec5a5cf8..f2d3d37326f5 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py @@ -5,9 +5,10 @@ from typing import Literal, Union +from pydantic.v1 import AnyUrl, BaseModel, Field + from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic.v1 import AnyUrl, BaseModel, Field class OAuthCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py index 7b497a6a9f35..4dfaddb5c4d5 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py @@ -5,10 +5,11 @@ import json -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from google.cloud import storage from google.oauth2 import credentials, service_account +from airbyte_cdk.sources.file_based.remote_file import RemoteFile + def get_gcs_client(config): if config.credentials.auth_type == "Service": diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py index 65038c7fa7ef..c0a00bf91939 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py @@ -7,9 +7,10 @@ import time import traceback +from orjson import orjson + from airbyte_cdk import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader from source_gcs.config_migrations import MigrateServiceAccount diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py index abc91843a449..b81e60e95a32 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py @@ -3,9 +3,10 @@ # from typing import Literal, Union -from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic.v1 import BaseModel, Field +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig + class OAuthCredentials(BaseModel): class Config(OneOfOptionConfig): diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py index fdd339c43424..53597e931640 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py @@ -11,14 +11,16 @@ import pytz import smart_open -from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from google.cloud import storage from google.oauth2 import credentials, service_account + +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from source_gcs.config import Config from source_gcs.helpers import GCSRemoteFile from source_gcs.zip_helper import ZipHelper + # google can raise warnings for end user credentials, wrapping it to Logger logging.captureWarnings(True) @@ -96,7 +98,6 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo last_modified = blob.updated.astimezone(pytz.utc).replace(tzinfo=None) if not start_date or last_modified >= start_date: - if self.config.credentials.auth_type == "Client": uri = f"gs://{blob.bucket.name}/{blob.name}" else: diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py b/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py index 43b7c2deeecf..609fc2e013dc 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py @@ -7,8 +7,10 @@ from typing import Iterable from google.cloud.storage.blob import Blob + from source_gcs.helpers import GCSRemoteFile + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py index 105a55e1ee08..5b72f1604164 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py @@ -6,10 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from source_gcs import Cursor, SourceGCSStreamReader from source_gcs.helpers import GCSRemoteFile +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig + @pytest.fixture def logger(): @@ -63,9 +64,10 @@ def zip_file(): uri=str(Path(__file__).parent / "resource/files/test.csv.zip"), last_modified=datetime.today(), mime_type=".zip", - displayed_uri="resource/files/test.csv.zip" + displayed_uri="resource/files/test.csv.zip", ) + @pytest.fixture def mocked_blob(): blob = Mock() diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py index ea301e8d9e12..eab24e5e8d14 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py @@ -7,4 +7,3 @@ def test_documentation_url(): assert "https" in Config.documentation_url() - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py index 142a7f8939c8..8812a03243ee 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py @@ -6,33 +6,42 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk import AirbyteEntrypoint from source_gcs import SourceGCS from source_gcs.config_migrations import MigrateServiceAccount +from airbyte_cdk import AirbyteEntrypoint + def load_config(path: str) -> Mapping[str, Any]: with open(path, "r") as f: return json.load(f) + def revert_config(path: str) -> None: migrated_config = load_config(path) del migrated_config["credentials"] with open(path, "w") as f: f.write(json.dumps(migrated_config)) + @pytest.mark.parametrize( "config_file_path, run_revert", [ # Migration is required (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_config.json"), True), # New config format - (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False) - ] + (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False), + ], ) def test_migrate_config(config_file_path, run_revert): args = ["check", "--config", config_file_path] - source = SourceGCS(MagicMock(), MagicMock, None, AirbyteEntrypoint.extract_config(args), None,) + source = SourceGCS( + MagicMock(), + MagicMock, + None, + AirbyteEntrypoint.extract_config(args), + None, + ) MigrateServiceAccount().migrate(args, source) migrated_config = load_config(config_file_path) @@ -43,4 +52,3 @@ def test_migrate_config(config_file_path, run_revert): if run_revert: revert_config(config_file_path) - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py index 59095eead97d..309f5167a14a 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py @@ -1,9 +1,10 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. from datetime import datetime +from source_gcs import Cursor + from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor -from source_gcs import Cursor def test_add_file_successfully(cursor, remote_file, logger): @@ -125,9 +126,6 @@ def test_add_file_zip_files(mocked_reader, zip_file, logger): cursor = Cursor(stream_config=FileBasedStreamConfig(name="test", globs=["**/*.zip"], format={"filetype": "csv"})) cursor.add_file(zip_file) - saved_history_cursor = datetime.strptime( - cursor._file_to_datetime_history[zip_file.displayed_uri], - cursor.DATE_TIME_FORMAT - ) + saved_history_cursor = datetime.strptime(cursor._file_to_datetime_history[zip_file.displayed_uri], cursor.DATE_TIME_FORMAT) assert saved_history_cursor == zip_file.last_modified diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py index 87b46cc061aa..a74903a226b3 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py @@ -3,10 +3,11 @@ from unittest.mock import patch import pytest -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from common import catalog_path, config_path from source_gcs.run import run +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + def test_run_with_non_existing_config(): with patch("sys.argv", ["", "check", "--config", "non_existing.json"]): diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py index 03062a560865..76c55728c0af 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py @@ -4,11 +4,12 @@ from unittest.mock import Mock import pytest -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.sources.file_based.availability_strategy import DefaultFileBasedAvailabilityStrategy from common import catalog_path, config_path from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.sources.file_based.availability_strategy import DefaultFileBasedAvailabilityStrategy + def _source_gcs(catalog, config): return SourceGCS( diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py index 163aaab29bf1..9ef22105964b 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py @@ -9,8 +9,15 @@ def test_transform_record(zip_file, mocked_reader, logger): stream = GCSStream( - config=Mock(), catalog_schema=Mock(), stream_reader=Mock(), availability_strategy=Mock(), discovery_policy=Mock(),parsers=Mock(), - validation_policy=Mock(), errors_collector=Mock(), cursor=Mock() + config=Mock(), + catalog_schema=Mock(), + stream_reader=Mock(), + availability_strategy=Mock(), + discovery_policy=Mock(), + parsers=Mock(), + validation_policy=Mock(), + errors_collector=Mock(), + cursor=Mock(), ) last_updated = zip_file.last_modified.isoformat() transformed_record = stream.transform_record({"field1": 1}, zip_file, last_updated) @@ -19,11 +26,7 @@ def test_transform_record(zip_file, mocked_reader, logger): assert transformed_record["_ab_source_file_url"] != zip_file.uri last_updated = datetime.today().isoformat() - csv_file = GCSRemoteFile( - uri="https://storage.googleapis.com/test/test", - last_modified=last_updated, - mime_type = "csv" - ) + csv_file = GCSRemoteFile(uri="https://storage.googleapis.com/test/test", last_modified=last_updated, mime_type="csv") transformed_record = stream.transform_record({"field1": 1}, csv_file, last_updated) assert transformed_record["_ab_source_file_url"] == csv_file.uri diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py index b5d5b501872e..b9508c8f5dde 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py @@ -4,19 +4,17 @@ from unittest.mock import Mock import pytest +from source_gcs import Config, SourceGCSStreamReader +from source_gcs.config import ServiceAccountCredentials + from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from source_gcs import Config, SourceGCSStreamReader -from source_gcs.config import ServiceAccountCredentials def test_get_matching_files_with_no_prefix(logger, mocked_reader): mocked_reader._config = Config( - credentials=ServiceAccountCredentials( - service_account='{"type": "service_account"}', - auth_type="Service" - ), + credentials=ServiceAccountCredentials(service_account='{"type": "service_account"}', auth_type="Service"), bucket="test_bucket", streams=[], ) diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py index d4ea6e2fb08f..f2595830b6db 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py @@ -6,7 +6,6 @@ def test_get_gcs_remote_files(mocked_blob, zip_file, caplog): - files = list(ZipHelper(mocked_blob, zip_file,tempfile.TemporaryDirectory()).get_gcs_remote_files()) + files = list(ZipHelper(mocked_blob, zip_file, tempfile.TemporaryDirectory()).get_gcs_remote_files()) assert len(files) == 1 assert "Picking up file test.csv from zip archive" in caplog.text - diff --git a/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-genesys/main.py b/airbyte-integrations/connectors/source-genesys/main.py index d34643d2aa21..fbd608c13d5a 100644 --- a/airbyte-integrations/connectors/source-genesys/main.py +++ b/airbyte-integrations/connectors/source-genesys/main.py @@ -4,5 +4,6 @@ from source_genesys.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py b/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py index 4a5adde124bc..67fae1b42130 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index 30d0d1106c43..27f2a07b5273 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -244,7 +245,6 @@ def path(self, **kwargs) -> str: class SourceGenesys(AbstractSource): def build_refresh_request_body(self) -> Mapping[str, Any]: - return { "grant_type": "client_credentials", "client_id": self.get_client_id(), @@ -259,7 +259,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - GENESYS_REGION_DOMAIN_MAP: Dict[str, str] = { "Americas (US East)": "mypurecloud.com", "Americas (US East 2)": "use2.us-gov-pure.cloud", diff --git a/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-github/fixtures/github.py b/airbyte-integrations/connectors/source-github/fixtures/github.py index 920b0677f952..ce0eea2a046c 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/github.py +++ b/airbyte-integrations/connectors/source-github/fixtures/github.py @@ -9,6 +9,7 @@ import requests + logging.basicConfig(level=logging.INFO) diff --git a/airbyte-integrations/connectors/source-github/fixtures/main.py b/airbyte-integrations/connectors/source-github/fixtures/main.py index 00468b290d42..47776cb04626 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/main.py +++ b/airbyte-integrations/connectors/source-github/fixtures/main.py @@ -8,6 +8,7 @@ from github import GitHubFiller + if __name__ == "__main__": api_token = sys.argv[1] repository = sys.argv[2] diff --git a/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py index 4d37ce6cccf5..bbe2a227e2f8 100644 --- a/airbyte-integrations/connectors/source-github/main.py +++ b/airbyte-integrations/connectors/source-github/main.py @@ -4,5 +4,6 @@ from source_github.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py b/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py index ac887892499c..f407b315b21f 100644 --- a/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py +++ b/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py @@ -6,6 +6,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py index 79ec73a9cd2f..f644b4c45266 100644 --- a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py +++ b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceGithub + logger = logging.getLogger("airbyte_logger") @@ -30,13 +31,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: @@ -95,12 +94,10 @@ def migrate(cls, args: List[str], source: SourceGithub) -> None: class MigrateRepository(MigrateStringToArray): - migrate_from_key: str = "repository" migrate_to_key: str = "repositories" class MigrateBranch(MigrateStringToArray): - migrate_from_key: str = "branch" migrate_to_key: str = "branches" diff --git a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py index 26abc891e897..ba36daec8ced 100644 --- a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py +++ b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING @@ -12,6 +13,7 @@ from . import constants + GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 401: ErrorResolution( response_action=ResponseAction.RETRY, diff --git a/airbyte-integrations/connectors/source-github/source_github/github_schema.py b/airbyte-integrations/connectors/source-github/source_github/github_schema.py index ace0d9c69e2d..bfa26d90dd37 100644 --- a/airbyte-integrations/connectors/source-github/source_github/github_schema.py +++ b/airbyte-integrations/connectors/source-github/source_github/github_schema.py @@ -6,6 +6,7 @@ import sgqlc.types.datetime import sgqlc.types.relay + github_schema = sgqlc.types.Schema() diff --git a/airbyte-integrations/connectors/source-github/source_github/graphql.py b/airbyte-integrations/connectors/source-github/source_github/graphql.py index 603e58f0182c..bdeb4655bcd8 100644 --- a/airbyte-integrations/connectors/source-github/source_github/graphql.py +++ b/airbyte-integrations/connectors/source-github/source_github/graphql.py @@ -11,6 +11,7 @@ from . import github_schema + _schema = github_schema _schema_root = _schema.github_schema @@ -165,7 +166,6 @@ def get_query_issue_reactions(owner, name, first, after, number=None): class QueryReactions: - # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request. # If we try to fetch too many (up to 100) we will spend too many scores of query cost. # https://docs.github.com/en/graphql/overview/resource-limitations#calculating-a-rate-limit-score-before-running-the-call diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 5698b8795833..9e6246e67ff3 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -60,7 +60,6 @@ class SourceGithub(AbstractSource): - continue_sync_on_stream_failure = True @staticmethod diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 7e9fcc80cc3e..cc8d8d8ee2b2 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk import BackoffStrategy, StreamSlice from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode from airbyte_cdk.models import Type as MessageType @@ -41,7 +42,6 @@ class GithubStreamABC(HttpStream, ABC): - primary_key = "id" # Detect streams with high API load @@ -80,7 +80,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} if next_page_token: @@ -756,7 +755,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class GitHubGraphQLStream(GithubStream, ABC): - http_method = "POST" def path( @@ -976,7 +974,6 @@ def request_body_json( class ReactionStream(GithubStream, CheckpointMixin, ABC): - parent_key = "id" copy_parent_key = "comment_id" cursor_field = "created_at" @@ -1394,9 +1391,9 @@ def _get_updated_state(self, current_stream_state: MutableMapping[str, Any], lat stream_state_value = current_stream_state.get(repository, {}).get(project_id, {}).get(column_id, {}).get(self.cursor_field) if stream_state_value: updated_state = max(updated_state, stream_state_value) - current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[ - self.cursor_field - ] = updated_state + current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[self.cursor_field] = ( + updated_state + ) return current_stream_state def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: @@ -1621,7 +1618,6 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, return record def get_error_handler(self) -> Optional[ErrorHandler]: - return ContributorActivityErrorHandler(logger=self.logger, max_retries=5, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING) def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]: diff --git a/airbyte-integrations/connectors/source-github/source_github/utils.py b/airbyte-integrations/connectors/source-github/source_github/utils.py index 3479e7c12b43..d9b7ea18d3bc 100644 --- a/airbyte-integrations/connectors/source-github/source_github/utils.py +++ b/airbyte-integrations/connectors/source-github/source_github/utils.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator @@ -99,7 +100,6 @@ def update_token(self) -> None: @property def token(self) -> str: - token = self.current_active_token return f"{self._auth_method} {token}" @@ -123,13 +123,15 @@ def _check_token_limits(self, token: str): ) token_info = self._tokens[token] remaining_info_core = rate_limit_info.get("core") - token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp( - remaining_info_core.get("reset") + token_info.count_rest, token_info.reset_at_rest = ( + remaining_info_core.get("remaining"), + pendulum.from_timestamp(remaining_info_core.get("reset")), ) remaining_info_graphql = rate_limit_info.get("graphql") - token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp( - remaining_info_graphql.get("reset") + token_info.count_graphql, token_info.reset_at_graphql = ( + remaining_info_graphql.get("remaining"), + pendulum.from_timestamp(remaining_info_graphql.get("reset")), ) def check_all_tokens(self): diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index f4e454dfab98..b2f4f27fe7a5 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -5,6 +5,7 @@ import pytest import responses + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py index bde678baef18..89c9b35fcc88 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py @@ -3,16 +3,18 @@ import json from unittest import TestCase +from source_github import SourceGithub + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from source_github import SourceGithub from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_repositories(["airbytehq/mock-test-0", "airbytehq/mock-test-1", "airbytehq/mock-test-2"]).build() @@ -162,12 +164,19 @@ def test_read_full_refresh_emits_per_partition_state(self): per_partition_state_1 = {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}} per_partition_state_2 = {"partition": {"repository": "airbytehq/mock-test-2"}, "cursor": {"__ab_full_refresh_sync_complete": True}} - incoming_state = StateBuilder().with_stream_state("assignees", { - "states": [ - {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - ] - }).build() + incoming_state = ( + StateBuilder() + .with_stream_state( + "assignees", + { + "states": [ + {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + ] + }, + ) + .build() + ) source = SourceGithub() actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), state=incoming_state) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py index 4140e6a1b63d..dfea868aed68 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py @@ -3,6 +3,8 @@ import json from unittest import TestCase, mock +from source_github import SourceGithub + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -10,10 +12,10 @@ from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStreamStatus, Level, TraceType -from source_github import SourceGithub from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_repositories(["airbytehq/integration-test"]).build() diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py index fdebc3af470b..06ad40cf2756 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py @@ -7,11 +7,13 @@ import os from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_github.config_migrations import MigrateBranch, MigrateRepository from source_github.source import SourceGithub +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py index ebe8eb56c4a4..829c6137eb95 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py @@ -8,13 +8,14 @@ import pendulum import pytest import responses -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType from freezegun import freeze_time from source_github import SourceGithub from source_github.streams import Organizations from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter, read_full_refresh +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + @responses.activate def test_multiple_tokens(rate_limit_mock_response): diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 388e726e7dde..8e0d2455d0fc 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -8,11 +8,12 @@ import pytest import responses -from airbyte_cdk.models import AirbyteConnectionStatus, Status -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_github import constants from source_github.source import SourceGithub +from airbyte_cdk.models import AirbyteConnectionStatus, Status +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + from .utils import command_check diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index d8944137c766..1ee7c865bbfb 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -10,10 +10,6 @@ import pytest import requests import responses -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction -from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException -from airbyte_protocol.models import FailureType from requests import HTTPError from responses import matchers from source_github import SourceGithub, constants @@ -55,8 +51,14 @@ ) from source_github.utils import read_full_refresh +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException +from airbyte_protocol.models import FailureType + from .utils import ProjectsResponsesAPI, read_incremental + DEFAULT_BACKOFF_DELAYS = [1, 2, 4, 8, 16] @@ -102,10 +104,34 @@ def test_backoff_time(time_mock, http_status, response_headers, expected_backoff @pytest.mark.parametrize( ("http_status", "response_headers", "text", "response_action", "error_message"), [ - (HTTPStatus.OK, {"X-RateLimit-Resource": "graphql"}, '{"errors": [{"type": "RATE_LIMITED"}]}', ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.OK}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"X-RateLimit-Remaining": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "60"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), + ( + HTTPStatus.OK, + {"X-RateLimit-Resource": "graphql"}, + '{"errors": [{"type": "RATE_LIMITED"}]}', + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.OK}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"X-RateLimit-Remaining": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "60"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), (HTTPStatus.INTERNAL_SERVER_ERROR, {}, "", ResponseAction.RETRY, "Internal server error."), (HTTPStatus.BAD_GATEWAY, {}, "", ResponseAction.RETRY, "Bad gateway."), (HTTPStatus.SERVICE_UNAVAILABLE, {}, "", ResponseAction.RETRY, "Service unavailable."), @@ -330,7 +356,6 @@ def test_stream_repositories_read(): @responses.activate @patch("time.sleep") def test_stream_projects_disabled(time_mock): - repository_args_with_start_date = {"start_date": "start_date", "page_size_for_large_streams": 30, "repositories": ["test_repo"]} stream = Projects(**repository_args_with_start_date) @@ -348,7 +373,6 @@ def test_stream_projects_disabled(time_mock): @responses.activate def test_stream_pull_requests_incremental_read(): - page_size = 2 repository_args_with_start_date = { "repositories": ["organization/repository"], @@ -412,7 +436,6 @@ def test_stream_pull_requests_incremental_read(): @responses.activate def test_stream_commits_incremental_read(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -451,18 +474,18 @@ def test_stream_commits_incremental_read(): "name": "branch", "commit": { "sha": "74445338726f0f8e1c27c10dce90ca00c5ae2858", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858", }, - "protected": False + "protected": False, }, { "name": "main", "commit": { "sha": "c27c10dce90ca00c5ae285874445338726f0f8e1", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1", }, - "protected": False - } + "protected": False, + }, ], status=200, ) @@ -503,7 +526,6 @@ def test_stream_commits_incremental_read(): @responses.activate def test_stream_pull_request_commits(): - repository_args = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -545,7 +567,6 @@ def test_stream_pull_request_commits(): @responses.activate def test_stream_project_columns(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -638,7 +659,6 @@ def test_stream_project_columns(): @responses.activate def test_stream_project_cards(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -734,7 +754,6 @@ def test_stream_project_cards(): @responses.activate def test_stream_comments(): - repository_args_with_start_date = { "repositories": ["organization/repository", "airbytehq/airbyte"], "page_size_for_large_streams": 2, @@ -866,7 +885,6 @@ def test_stream_comments(): @responses.activate def test_streams_read_full_refresh(): - repository_args = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -927,7 +945,6 @@ def get_records(cursor_field): @responses.activate def test_stream_reviews_incremental_read(): - repository_args_with_start_date = { "start_date": "2000-01-01T00:00:00Z", "page_size_for_large_streams": 30, @@ -1002,7 +1019,6 @@ def test_stream_team_members_full_refresh(time_mock, caplog, rate_limit_mock_res @responses.activate def test_stream_commit_comment_reactions_incremental_read(): - repository_args = {"repositories": ["airbytehq/integration-test"], "page_size_for_large_streams": 100} stream = CommitCommentReactions(**repository_args) stream._parent_stream._http_client._session.cache.clear() @@ -1083,7 +1099,6 @@ def test_stream_commit_comment_reactions_incremental_read(): @responses.activate def test_stream_workflow_runs_read_incremental(monkeypatch): - repository_args_with_start_date = { "repositories": ["org/repos"], "page_size_for_large_streams": 30, @@ -1202,7 +1217,6 @@ def test_stream_workflow_runs_read_incremental(monkeypatch): @responses.activate def test_stream_workflow_jobs_read(): - repository_args = { "repositories": ["org/repo"], "page_size_for_large_streams": 100, @@ -1303,7 +1317,6 @@ def test_stream_workflow_jobs_read(): @responses.activate def test_stream_pull_request_comment_reactions_read(): - repository_args_with_start_date = { "start_date": "2022-01-01T00:00:00Z", "page_size_for_large_streams": 2, @@ -1361,11 +1374,7 @@ def test_stream_projects_v2_graphql_retry(time_mock, rate_limit_mock_response): } stream = ProjectsV2(**repository_args_with_start_date) resp = responses.add( - responses.POST, - "https://api.github.com/graphql", - json={"errors": "not found"}, - status=200, - headers={'Retry-After': '5'} + responses.POST, "https://api.github.com/graphql", json={"errors": "not found"}, status=200, headers={"Retry-After": "5"} ) backoff_strategy = GithubStreamABCBackoffStrategy(stream) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/utils.py b/airbyte-integrations/connectors/source-github/unit_tests/utils.py index 81b32a929bcf..77472173550b 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/utils.py @@ -6,6 +6,7 @@ from unittest import mock import responses + from airbyte_cdk.models import SyncMode from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification from airbyte_cdk.sources import Source diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gitlab/main.py b/airbyte-integrations/connectors/source-gitlab/main.py index 1c322c2f2c48..8dcc7a60745f 100644 --- a/airbyte-integrations/connectors/source-gitlab/main.py +++ b/airbyte-integrations/connectors/source-gitlab/main.py @@ -4,5 +4,6 @@ from source_gitlab.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py index ec47f547f591..1640cc68ad99 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceGitlab + logger = logging.getLogger("airbyte_logger") @@ -30,13 +31,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py index c6f180e38de9..1b4a6d60636d 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -6,9 +6,11 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.sources.streams import Stream from source_gitlab.source import SourceGitlab +from airbyte_cdk.sources.streams import Stream + + BASE_CONFIG = { "start_date": "2021-01-01T00:00:00Z", "api_url": "gitlab.com", diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py index b61fb232906d..4293f2ed4695 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py @@ -5,6 +5,7 @@ from source_gitlab.config_migrations import MigrateGroups from source_gitlab.source import SourceGitlab + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py index 4e7e0b40483f..7222619f32cc 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py @@ -3,9 +3,10 @@ # +from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name class TestGroupStreamsPartitionRouter: @@ -56,9 +57,7 @@ def test_projects_stream_slices_with_group_project_ids(self, requests_mock): url=f"https://gitlab.com/api/v4/groups/{group_id}?per_page=50", json=[{"id": group_id, "projects": [{"id": project_id, "path_with_namespace": project_id}]}], ) - expected_stream_slices.append( - StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={}) - ) + expected_stream_slices.append(StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})) requests_mock.get(url=GROUPS_LIST_URL, json=groups_list_response) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index b650c9e52b6d..06f55edf7d51 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -6,9 +6,10 @@ import logging import pytest -from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from source_gitlab import SourceGitlab +from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream + def test_streams(config): source = SourceGitlab() @@ -19,9 +20,7 @@ def test_streams(config): def test_connection_success(config, requests_mock): requests_mock.get(url="/api/v4/groups", json=[{"id": "g1"}]) - requests_mock.get( - url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}] - ) + requests_mock.get(url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]) requests_mock.get(url="/api/v4/projects/p1", json={"id": "p1"}) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) @@ -30,9 +29,7 @@ def test_connection_success(config, requests_mock): def test_connection_invalid_projects_and_projects(config_with_project_groups, requests_mock): requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1?per_page=50", status_code=404) - requests_mock.register_uri( - "GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404 - ) + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404) requests_mock.register_uri("GET", "https://gitlab.com/api/v4/projects/p1?per_page=50&statistics=1", status_code=404) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config_with_project_groups) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index 9eef564ef235..820ca47f17b5 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -4,9 +4,11 @@ import pytest -from airbyte_cdk.models import SyncMode from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name +from airbyte_cdk.models import SyncMode + + CONFIG = BASE_CONFIG | {"projects_list": ["p_1"]} @@ -123,7 +125,7 @@ def test_should_retry(requests_mock, stream_name, extra_mocks): "user_full_name": "John", "environment_name": "dev", "project_id": "p_1", - } + }, ), ( "merge_request_commits", @@ -157,11 +159,12 @@ def test_stream_slices_child_stream(requests_mock): url="https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1", json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}], ) - stream_state = {"13082000": {""'created_at': "2021-03-10T23:58:1213"}} + stream_state = {"13082000": {"" "created_at": "2021-03-10T23:58:1213"}} slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state)) assert slices + def test_request_params(): commits = get_stream_by_name("commits", CONFIG) assert commits.retriever.requester.get_request_params() == {"with_stats": "true"} diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py index 5019d87b7b44..a347efae2f82 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py @@ -11,7 +11,7 @@ ("http://example", (True, "http", "example")), ("test://example.com", (False, "", "")), ("https://example.com/test/test2", (False, "", "")), - ) + ), ) def test_parse_url(url, expected): assert parse_url(url) == expected diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gnews/components.py b/airbyte-integrations/connectors/source-gnews/components.py index df2f0a450bda..d93c3d963174 100644 --- a/airbyte-integrations/connectors/source-gnews/components.py +++ b/airbyte-integrations/connectors/source-gnews/components.py @@ -7,6 +7,7 @@ from typing import Any, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.requesters.error_handlers import BackoffStrategy from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py index f2b023043a0b..de602c599148 100644 --- a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py +++ b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py @@ -4,9 +4,10 @@ from json import dumps, load from typing import Dict +from components import IncrementalSingleBodyFilterCursor + from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption -from components import IncrementalSingleBodyFilterCursor def config() -> Dict[str, str]: @@ -17,35 +18,30 @@ def config() -> Dict[str, str]: class TestGetRequestFilterOptions(unittest.TestCase): - def setUp(self): self.instance = IncrementalSingleBodyFilterCursor( - start_datetime= MinMaxDatetime(datetime= "2024-03-25", datetime_format= "%Y-%m-%dT%H:%M:%SZ", parameters = None), - cursor_field = "reviewTime", - datetime_format= "%Y-%m-%d", - config= config, - parameters = None + start_datetime=MinMaxDatetime(datetime="2024-03-25", datetime_format="%Y-%m-%dT%H:%M:%SZ", parameters=None), + cursor_field="reviewTime", + datetime_format="%Y-%m-%d", + config=config, + parameters=None, ) - + def test_get_request_filter_options_no_stream_slice(self): expected = {} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, None) assert result == expected - + def test_get_request_filter_options_with_start_time_option(self): - expected = {'filter': {'reviewFromDate': '2024-03-25'}} - - self.instance.start_time_option = RequestOption( - inject_into = "body_json", - field_name = "filter, reviewFromDate", - parameters = None - ) - self.instance.stream_slice = {'start_time': '2024-03-25', 'end_time': '2024-03-29'} + expected = {"filter": {"reviewFromDate": "2024-03-25"}} + + self.instance.start_time_option = RequestOption(inject_into="body_json", field_name="filter, reviewFromDate", parameters=None) + self.instance.stream_slice = {"start_time": "2024-03-25", "end_time": "2024-03-29"} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, self.instance.stream_slice) assert result == expected -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py index c6960355b90c..aa6c453f9018 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py @@ -6,10 +6,11 @@ from pathlib import Path import pytest -from airbyte_cdk.models import SyncMode from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow from source_google_ads.source import SourceGoogleAds +from airbyte_cdk.models import SyncMode + @pytest.fixture(scope="module") def config(): diff --git a/airbyte-integrations/connectors/source-google-ads/main.py b/airbyte-integrations/connectors/source-google-ads/main.py index 2824c4955943..7e8e4ddf5226 100644 --- a/airbyte-integrations/connectors/source-google-ads/main.py +++ b/airbyte-integrations/connectors/source-google-ads/main.py @@ -4,5 +4,6 @@ from source_google_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py index 81bae9ceb5cb..61990be3b7b9 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py @@ -14,6 +14,7 @@ from .utils import GAQL + FULL_REFRESH_CUSTOM_TABLE = [ "asset", "asset_group_listing_group_filter", diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 4a3ac096cfdd..4b2a25a08a32 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -9,6 +9,7 @@ from .streams import GoogleAdsStream, IncrementalGoogleAdsStream from .utils import GAQL + DATE_TYPES = ("segments.date", "segments.month", "segments.quarter", "segments.week") diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 3a2128f6182f..982210066766 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -7,8 +7,6 @@ from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping import backoff -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.client import GoogleAdsClient from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse from google.api_core.exceptions import InternalServerError, ServerError, TooManyRequests @@ -17,8 +15,12 @@ from google.protobuf.message import Message from proto.marshal.collections import Repeated, RepeatedComposite +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + from .utils import logger + API_VERSION = "v17" diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 562dacf6f899..4fa488df17e3 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -6,11 +6,12 @@ import logging from typing import Any, Iterable, List, Mapping, MutableMapping, Tuple +from pendulum import duration, parse, today + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from pendulum import duration, parse, today from .custom_query_stream import CustomQuery, IncrementalCustomQuery from .google_ads import GoogleAds diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 0dc3374548fa..99da0e71f661 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -8,15 +8,16 @@ import backoff import pendulum -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.sources.streams import CheckpointMixin, Stream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.services.services.google_ads_service.pagers import SearchPager from google.ads.googleads.v17.services.types.google_ads_service import SearchGoogleAdsResponse from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests, Unauthenticated +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.sources.streams import CheckpointMixin, Stream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException + from .google_ads import GoogleAds, logger from .models import CustomerModel from .utils import ExpiredPageTokenError, chunk_date_range, detached, generator_backoff, get_resource_name, parse_dates, traced_exception diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index 824d9c9cb470..5b31eac22e37 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -13,8 +13,6 @@ from typing import Any, Callable, Generator, Iterable, MutableMapping, Optional, Tuple, Type, Union import pendulum -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.errors.types.authentication_error import AuthenticationErrorEnum from google.ads.googleads.v17.errors.types.authorization_error import AuthorizationErrorEnum @@ -23,6 +21,10 @@ from google.ads.googleads.v17.errors.types.request_error import RequestErrorEnum from google.api_core.exceptions import Unauthenticated +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py index f409e0f1ac52..21623048feda 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -55,6 +55,7 @@ def mock_oauth_call(requests_mock): def customers(config): return [CustomerModel(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + @pytest.fixture def additional_customers(config, customers): return customers + [CustomerModel(id="789", time_zone="local", is_manager_account=False)] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py index a65712bf0e9d..c6a7e287ed07 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_ads.config_migrations import MigrateCustomQuery from source_google_ads.source import SourceGoogleAds +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py index 49679c3be829..bb99b9cc427f 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -8,12 +8,13 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads.google_ads import GoogleAds from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds from source_google_ads.streams import AdGroupLabel, Label, ServiceAccounts +from airbyte_cdk.utils import AirbyteTracedException + from .common import MockGoogleAdsClient, mock_google_ads_request_failure @@ -35,7 +36,10 @@ def mock_get_customers(mocker): "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", ), (["QUERY_ERROR"], "Incorrect custom query. Error in query: unexpected end of query."), - (["UNRECOGNIZED_FIELD"], "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid."), + ( + ["UNRECOGNIZED_FIELD"], + "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid.", + ), ( ["RESOURCE_EXHAUSTED"], ( diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index e3bec07f2976..5c8f93b4e7cf 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -8,14 +8,16 @@ import pendulum import pytest -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow from google.auth import exceptions from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import chunk_date_range +from airbyte_cdk.utils import AirbyteTracedException + from .common import MockGoogleAdsClient, MockGoogleAdsService + SAMPLE_SCHEMA = { "properties": { "segment.date": { @@ -148,11 +150,12 @@ def test_get_field_value(): response = GoogleAds.get_field_value(MockedDateSegment(date), field, {}) assert response == date + def test_get_field_value_object(): expected_response = [ - { "text": "An exciting headline", "policySummaryInfo": { "reviewStatus": "REVIEWED","approvalStatus":"APPROVED" } }, - { "text": "second" }, - ] + {"text": "An exciting headline", "policySummaryInfo": {"reviewStatus": "REVIEWED", "approvalStatus": "APPROVED"}}, + {"text": "second"}, + ] field = "ad_group_ad.ad.responsive_search_ad.headlines" ads_row = GoogleAdsRow( ad_group_ad={ @@ -161,14 +164,9 @@ def test_get_field_value_object(): "headlines": [ { "text": "An exciting headline", - "policy_summary_info": { - "review_status": "REVIEWED", - "approval_status": "APPROVED" - } + "policy_summary_info": {"review_status": "REVIEWED", "approval_status": "APPROVED"}, }, - { - "text": "second" - } + {"text": "second"}, ] } } @@ -176,13 +174,14 @@ def test_get_field_value_object(): ) response = GoogleAds.get_field_value(ads_row, field, {}) - assert [json.loads(i) for i in response] == expected_response + assert [json.loads(i) for i in response] == expected_response + def test_get_field_value_strings(): expected_response = [ - "http://url_one.com", - "https://url_two.com", - ] + "http://url_one.com", + "https://url_two.com", + ] ads_row = GoogleAdsRow( ad_group_ad={ "ad": { @@ -197,6 +196,7 @@ def test_get_field_value_strings(): response = GoogleAds.get_field_value(ads_row, field, {}) assert response == expected_response + def test_parse_single_result(): date = "2001-01-01" response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py index 34b89d1e2e88..50173bce7ff3 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py @@ -8,11 +8,12 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import CampaignCriterion, ChangeStatus +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + def mock_response_parent(): yield [ @@ -82,7 +83,7 @@ def test_change_status_stream(config, customers): def test_change_status_stream_slices(config, additional_customers): - """ Change status stream slices should return correct empty slices for the new customers """ + """Change status stream slices should return correct empty slices for the new customers""" google_api = MockGoogleAds(credentials=config["credentials"]) stream = ChangeStatus(api=google_api, customers=additional_customers) @@ -94,12 +95,19 @@ def test_change_status_stream_slices(config, additional_customers): result_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(result_slices) == 2 - assert result_slices == [{'start_date': '2023-11-01 12:36:04.772447', 'end_date': '2023-11-02 00:00:00.000000', 'customer_id': '123', - 'login_customer_id': None}, {'customer_id': '789', 'login_customer_id': None}] + assert result_slices == [ + { + "start_date": "2023-11-01 12:36:04.772447", + "end_date": "2023-11-02 00:00:00.000000", + "customer_id": "123", + "login_customer_id": None, + }, + {"customer_id": "789", "login_customer_id": None}, + ] def test_incremental_events_stream_slices(config, additional_customers): - """ Test if the empty slice will be produced for the new customers """ + """Test if the empty slice will be produced for the new customers""" stream_state = {"change_status": {"123": {"change_status.last_change_date_time": "2023-06-12 13:20:01.003295"}}} google_api = MockGoogleAds(credentials=config["credentials"]) @@ -121,12 +129,21 @@ def test_incremental_events_stream_slices(config, additional_customers): stream_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(stream_slices) == 2 - assert stream_slices == [{'customer_id': '123', 'updated_ids': {'2', '1'}, 'deleted_ids': {'3', '4'}, - 'record_changed_time_map': {'1': '2023-06-13 12:36:01.772447', '2': '2023-06-13 12:36:02.772447', - '3': '2023-06-13 12:36:03.772447', '4': '2023-06-13 12:36:04.772447'}, - 'login_customer_id': None}, - {'customer_id': '789', 'updated_ids': set(), 'deleted_ids': set(), 'record_changed_time_map': {}, - 'login_customer_id': None}] + assert stream_slices == [ + { + "customer_id": "123", + "updated_ids": {"2", "1"}, + "deleted_ids": {"3", "4"}, + "record_changed_time_map": { + "1": "2023-06-13 12:36:01.772447", + "2": "2023-06-13 12:36:02.772447", + "3": "2023-06-13 12:36:03.772447", + "4": "2023-06-13 12:36:04.772447", + }, + "login_customer_id": None, + }, + {"customer_id": "789", "updated_ids": set(), "deleted_ids": set(), "record_changed_time_map": {}, "login_customer_id": None}, + ] def test_child_incremental_events_read(config, customers): diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index fd53878536cf..10c384e532c9 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -10,8 +10,6 @@ import pendulum import pytest -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, FailureType, SyncMode from pendulum import duration, today from source_google_ads.custom_query_stream import IncrementalCustomQuery from source_google_ads.google_ads import GoogleAds @@ -20,6 +18,9 @@ from source_google_ads.streams import AdGroupAdLegacy, chunk_date_range from source_google_ads.utils import GAQL +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, FailureType, SyncMode + from .common import MockGoogleAdsClient diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index d0665bea9ac9..fbf8c8f07f3c 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -6,8 +6,6 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure from google.ads.googleads.v17.errors.types.request_error import RequestErrorEnum @@ -16,6 +14,10 @@ from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import AdGroup, ClickView, Customer, CustomerLabel +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException + + # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired. exception = GoogleAdsException( error=RpcError(), @@ -251,8 +253,10 @@ def test_retry_500_raises_transient_error(mocker, config, customers): with pytest.raises(AirbyteTracedException) as exception: records = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) - assert exception.value.internal_message == ("Internal Error encountered Unable to fetch data from Google Ads API due to " - "temporal error on the Google Ads server. Please retry again later. ") + assert exception.value.internal_message == ( + "Internal Error encountered Unable to fetch data from Google Ads API due to " + "temporal error on the Google Ads server. Please retry again later. " + ) assert exception.value.failure_type == FailureType.transient_error assert mocked_search.call_count == 5 assert records == [] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py index fe162fb993bc..5fbeab771444 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -8,10 +8,11 @@ import backoff import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads import SourceGoogleAds from source_google_ads.utils import GAQL, generator_backoff +from airbyte_cdk.utils import AirbyteTracedException + def test_parse_GAQL_ok(): sql = GAQL.parse("SELECT field FROM table") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py index 93839ed0e51a..613303154767 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py @@ -4,5 +4,6 @@ from source_google_analytics_data_api.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py index 27591cb3a1a7..2267cd3a19e3 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py @@ -8,9 +8,10 @@ from typing import Any, Iterable, Mapping, Optional import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from requests.exceptions import JSONDecodeError +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + from .utils import API_LIMIT_PER_HOUR diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py index 4a4bedce6ac3..b290368c983f 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py @@ -6,6 +6,7 @@ import jwt import requests + from source_google_analytics_data_api import utils diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py index c055b6445bc2..e1d1d23ccf82 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py @@ -8,6 +8,7 @@ import dpath.util import orjson + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import AirbyteMessageSerializer @@ -15,6 +16,7 @@ from .source import SourceGoogleAnalyticsDataApi + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py index ea17f13864ab..f3e4723157c7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + PROPERTY_ID_DOCS_URL = "https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id" MESSAGE = "Incorrect Property ID: {property_id}. Access was denied to the property ID entered. Check your access to the Property ID or use Google Analytics {property_id_docs_url} to find your Property ID." diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 957805ecde5b..1ed3e035984c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -18,6 +18,8 @@ import jsonschema import pendulum import requests +from requests import HTTPError + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -28,7 +30,6 @@ from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError from source_google_analytics_data_api import utils from source_google_analytics_data_api.google_analytics_data_api_base_error_mapping import get_google_analytics_data_api_base_error_mapping from source_google_analytics_data_api.google_analytics_data_api_metadata_error_mapping import ( @@ -49,6 +50,7 @@ transform_json, ) + # set the quota handler globally since limitations are the same for all streams # the initial values should be saved once and tracked for each stream, inclusively. GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py index d2783f6634ba..f08a5472c0d4 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py @@ -12,9 +12,11 @@ import jsonschema import pandas as pd + from airbyte_cdk.sources.streams.http import requests_native_auth as auth from source_google_analytics_data_api.authenticator import GoogleServiceKeyAuthenticator + DATE_FORMAT = "%Y-%m-%d" metrics_data_native_types_map: Dict = { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py index fcd8e1b879be..95b944b3bae7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pytest + # json credentials with fake private key json_credentials = """ { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py index bd5af212c081..af86742f72eb 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py @@ -4,9 +4,11 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from source_google_analytics_data_api.api_quota import GoogleAnalyticsApiQuota +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + + TEST_QUOTA_INSTANCE: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() @@ -35,7 +37,7 @@ def test_check_initial_quota_is_empty(): "potentiallyThresholdedRequestsPerHour": {"consumed": 1, "remaining": 26}, } }, - False, # partial_quota + False, # partial_quota ResponseAction.RETRY, None, # backoff_time_exp False, # stop_iter_exp diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py index a0df1b3f59d2..aa70ba13cb9d 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py @@ -4,10 +4,11 @@ from unittest.mock import patch -from airbyte_cdk.entrypoint import AirbyteEntrypoint from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi from source_google_analytics_data_api.config_migrations import MigratePropertyID +from airbyte_cdk.entrypoint import AirbyteEntrypoint + @patch.object(SourceGoogleAnalyticsDataApi, "read_config") @patch.object(SourceGoogleAnalyticsDataApi, "write_config") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py index 340d6db7660e..aac51ada0dcc 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py @@ -8,11 +8,13 @@ from typing import Any, Mapping import dpath.util -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_analytics_data_api.config_migrations import MigrateCustomReportsCohortSpec from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py index 5bee2f8ab6b9..39b569b5f234 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_analytics_data_api.config_migrations import MigrateCustomReports from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index e86201860e8b..d2d801e9bb2d 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -5,14 +5,15 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status -from airbyte_cdk.sources.streams.http.http import HttpStatusErrorHandler -from airbyte_cdk.utils import AirbyteTracedException from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi from source_google_analytics_data_api.api_quota import GoogleAnalyticsApiQuotaBase from source_google_analytics_data_api.source import GoogleAnalyticsDatApiErrorHandler, MetadataDescriptor from source_google_analytics_data_api.utils import NO_DIMENSIONS, NO_METRICS, NO_NAME, WRONG_CUSTOM_REPORT_CONFIG, WRONG_JSON_SYNTAX +from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status +from airbyte_cdk.sources.streams.http.http import HttpStatusErrorHandler +from airbyte_cdk.utils import AirbyteTracedException + @pytest.mark.parametrize( "config_values, is_successful, message", @@ -111,32 +112,42 @@ def test_check_failure_throws_exception(requests_mock, config_gen, error_code): def test_exhausted_quota_recovers_after_two_retries(requests_mock, config_gen): """ - If the account runs out of quota the api will return a message asking us to back off for one hour. - We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not - it will backoff again 30 minutes and quote should be reestablished by then. - Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota - recovery time and also the backoff time. + If the account runs out of quota the api will return a message asking us to back off for one hour. + We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not + it will backoff again 30 minutes and quote should be reestablished by then. + Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota + recovery time and also the backoff time. """ requests_mock.register_uri( "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} ) - error_response = {"error": {"message":"Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see"}} + error_response = { + "error": { + "message": "Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see" + } + } requests_mock.register_uri( "GET", "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata", # first try we get 429 t=~0 - [{"json": error_response, "status_code": 429}, - # first retry we get 429 t=~1800 - {"json": error_response, "status_code": 429}, - # second retry quota is recovered, t=~3600 - {"json": { - "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], - "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], - }, "status_code": 200} - ] + [ + {"json": error_response, "status_code": 429}, + # first retry we get 429 t=~1800 + {"json": error_response, "status_code": 429}, + # second retry quota is recovered, t=~3600 + { + "json": { + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + "status_code": 200, + }, + ], ) + def fix_time(time): - return int(time / 600 ) + return int(time / 600) + source = SourceGoogleAnalyticsDataApi() logger = MagicMock() max_time_fixed = fix_time(GoogleAnalyticsDatApiErrorHandler.QUOTA_RECOVERY_TIME) @@ -146,10 +157,18 @@ def fix_time(time): potentially_thresholded_requests_per_hour_mapping_fixed = { **potentially_thresholded_requests_per_hour_mapping, "backoff": fixed_threshold_backoff_time, - } + } with ( - patch.object(GoogleAnalyticsDatApiErrorHandler, 'QUOTA_RECOVERY_TIME', new=max_time_fixed), - patch.object(GoogleAnalyticsApiQuotaBase, 'quota_mapping', new={**GoogleAnalyticsApiQuotaBase.quota_mapping,"potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed})): + patch.object(GoogleAnalyticsDatApiErrorHandler, "QUOTA_RECOVERY_TIME", new=max_time_fixed), + patch.object( + GoogleAnalyticsApiQuotaBase, + "quota_mapping", + new={ + **GoogleAnalyticsApiQuotaBase.quota_mapping, + "potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed, + }, + ), + ): output = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert output == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) @@ -164,7 +183,7 @@ def test_check_failure(requests_mock, config_gen, error_code): ) source = SourceGoogleAnalyticsDataApi() logger = MagicMock() - with patch.object(HttpStatusErrorHandler, 'max_retries', new=0): + with patch.object(HttpStatusErrorHandler, "max_retries", new=0): airbyte_status = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert airbyte_status.status == Status.FAILED assert airbyte_status.message == repr("Failed to get metadata, over quota, try later") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 3ab018bbe6d3..97cb79d8b64d 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -10,11 +10,12 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction from freezegun import freeze_time from requests.models import Response from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream, SourceGoogleAnalyticsDataApi +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction + from .utils import read_incremental @@ -85,10 +86,9 @@ def test_request_body_json(patch_base_class): {"name": "browser"}, ], "keepEmptyRows": True, - "dateRanges": [{ - "startDate": request_body_params["stream_slice"]["startDate"], - "endDate": request_body_params["stream_slice"]["endDate"] - }], + "dateRanges": [ + {"startDate": request_body_params["stream_slice"]["startDate"], "endDate": request_body_params["stream_slice"]["endDate"]} + ], "returnPropertyQuota": True, "offset": str(0), "limit": "100000", @@ -269,8 +269,8 @@ def test_should_retry(patch_base_class, http_status, response_action_expected, r if response_body: json_data = response_body response_mock._content = str.encode(json.dumps(json_data)) - response_mock.headers = {'Content-Type': 'application/json'} - response_mock.encoding = 'utf-8' + response_mock.headers = {"Content-Type": "application/json"} + response_mock.encoding = "utf-8" stream = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]) assert stream.get_error_handler().interpret_response(response_mock).response_action == response_action_expected @@ -312,6 +312,7 @@ def test_stream_slices(): {"startDate": "2022-12-30", "endDate": "2023-01-01"}, ] + @freeze_time("2023-01-01 00:00:00") def test_full_refresh(): """ @@ -319,9 +320,7 @@ def test_full_refresh(): """ config = {"date_ranges_start_date": datetime.date(2022, 12, 29), "window_in_days": 1, "dimensions": ["browser", "country", "language"]} stream = GoogleAnalyticsDataApiBaseStream(authenticator=None, config=config) - full_refresh_state = { - "__ab_full_refresh_state_message": True - } + full_refresh_state = {"__ab_full_refresh_state_message": True} slices = list(stream.stream_slices(sync_mode=None, stream_state=full_refresh_state)) assert slices == [ {"startDate": "2022-12-29", "endDate": "2022-12-29"}, @@ -423,47 +422,37 @@ def test_read_incremental(requests_mock): {"property_id": 123, "yearWeek": "202202", "totalUsers": 140, "startDate": "2022-01-10", "endDate": "2022-01-10"}, ] + @pytest.mark.parametrize( "config_dimensions, expected_state", [ pytest.param(["browser", "country", "language", "date"], {"date": "20240320"}, id="test_date_no_cursor_field_dimension"), pytest.param(["browser", "country", "language"], {}, id="test_date_cursor_field_dimension"), - ] + ], ) def test_get_updated_state(config_dimensions, expected_state): config = { - "credentials": { - "auth_type": "Service", - "credentials_json": "{ \"client_email\": \"a@gmail.com\", \"client_id\": \"1234\", \"client_secret\": \"5678\", \"private_key\": \"5678\"}" - }, - "date_ranges_start_date": "2023-04-01", - "window_in_days": 30, - "property_ids": ["123"], - "custom_reports_array": [ - { - "name": "pivot_report", - "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], - "dimensions": config_dimensions, - "metrics": ["sessions"], - "pivots": [ - { - "fieldNames": ["browser"], - "limit": 5 - }, - { - "fieldNames": ["country"], - "limit": 250 - }, + "credentials": { + "auth_type": "Service", + "credentials_json": '{ "client_email": "a@gmail.com", "client_id": "1234", "client_secret": "5678", "private_key": "5678"}', + }, + "date_ranges_start_date": "2023-04-01", + "window_in_days": 30, + "property_ids": ["123"], + "custom_reports_array": [ { - "fieldNames": ["language"], - "limit": 15 + "name": "pivot_report", + "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], + "dimensions": config_dimensions, + "metrics": ["sessions"], + "pivots": [ + {"fieldNames": ["browser"], "limit": 5}, + {"fieldNames": ["country"], "limit": 250}, + {"fieldNames": ["language"], "limit": 15}, + ], + "cohortSpec": {"enabled": "false"}, } - ], - "cohortSpec": { - "enabled": "false" - } - } - ] + ], } source = SourceGoogleAnalyticsDataApi() config = source._validate_and_transform(config, report_names=set()) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py index 4d84f48a074b..d2371e025304 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py @@ -10,6 +10,7 @@ from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type from airbyte_cdk.utils import AirbyteTracedException + if __name__ == "__main__": logger = init_logger("airbyte") init_uncaught_exception_handler(logger) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/main.py b/airbyte-integrations/connectors/source-google-analytics-v4/main.py index 4d84f48a074b..d2371e025304 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/main.py @@ -10,6 +10,7 @@ from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type from airbyte_cdk.utils import AirbyteTracedException + if __name__ == "__main__": logger = init_logger("airbyte") init_uncaught_exception_handler(logger) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py index 9e6d232996e6..6f0b224882f3 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, ValidationError, validator from pydantic.class_validators import root_validator + ERROR_MSG_MISSING_SEGMENT_DIMENSION = "errors: `ga:segment` is required" @@ -99,7 +100,6 @@ def explain(self, errors: List[Dict]): @dataclass class CustomReportsValidator: - custom_reports: Union[List[Dict], Dict] = Field(default_factory=list) def __post_init__(self): @@ -108,7 +108,6 @@ def __post_init__(self): self.explainer: Explainer = Explainer() def validate(self): - # local import of airbyte_cdk dependencies from airbyte_cdk.models import FailureType from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index a71786d1a3a9..5bfa7b4cf297 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -14,6 +14,7 @@ import jwt import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -23,6 +24,7 @@ from .custom_reports_validator import CustomReportsValidator + DATA_IS_NOT_GOLDEN_MSG = "Google Analytics data is not golden. Future requests may return different data." RESULT_IS_SAMPLED_MSG = ( @@ -179,7 +181,6 @@ def raise_on_http_errors(self) -> bool: def request_body_json( self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs: Any ) -> Optional[Mapping]: - metrics = [{"expression": metric} for metric in self.metrics] dimensions = [{"name": dimension} for dimension in self.dimensions] segments = [{"segmentId": segment} for segment in self.segments] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py index 35e9d17716d8..58cb13723d30 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pendulum import pytest + from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.sources.streams.http.auth import NoAuth diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py index 4becd6c2d323..023e077dfd44 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_google_analytics_v4.custom_reports_validator import CustomReportsValidator +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @pytest.mark.parametrize( "custom_reports, expected", diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index a4a9f276ba14..bd6cdc1ac209 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -8,7 +8,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode, Type from freezegun import freeze_time from source_google_analytics_v4.source import ( DATA_IS_NOT_GOLDEN_MSG, @@ -19,6 +18,9 @@ SourceGoogleAnalyticsV4, ) +from airbyte_cdk.models import SyncMode, Type + + expected_metrics_dimensions_type_map = ( {"ga:users": "INTEGER", "ga:newUsers": "INTEGER"}, {"ga:date": "STRING", "ga:country": "STRING"}, diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-directory/main.py b/airbyte-integrations/connectors/source-google-directory/main.py index fa60e31af90e..62faf30d2f24 100644 --- a/airbyte-integrations/connectors/source-google-directory/main.py +++ b/airbyte-integrations/connectors/source-google-directory/main.py @@ -4,5 +4,6 @@ from source_google_directory.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py index 6a74d4cd0d75..9151f37a94bc 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py @@ -17,6 +17,7 @@ from .utils import rate_limit_handling + SCOPES = ["https://www.googleapis.com/auth/admin.directory.user.readonly", "https://www.googleapis.com/auth/admin.directory.group.readonly"] diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py index 6b0c294530cd..abe3d6fde3ed 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py @@ -7,6 +7,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-drive/main.py b/airbyte-integrations/connectors/source-google-drive/main.py index 606e4f7641e8..53fba38d7846 100644 --- a/airbyte-integrations/connectors/source-google-drive/main.py +++ b/airbyte-integrations/connectors/source-google-drive/main.py @@ -4,5 +4,6 @@ from source_google_drive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py index 95c7572471c6..fdecc52aab91 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Union import dpath.util +from pydantic import BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from pydantic import BaseModel, Field class OAuthCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py index ac73c84b5a19..3cf03d175909 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -10,16 +10,18 @@ from io import IOBase from typing import Iterable, List, Optional, Set -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from google.oauth2 import credentials, service_account from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload + +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_google_drive.utils import get_folder_id from .spec import SourceGoogleDriveSpec + FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" GOOGLE_DOC_MIME_TYPE = "application/vnd.google-apps.document" EXPORTABLE_DOCUMENTS_MIME_TYPES = [ diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py index a98f2436cae4..66fce445ada7 100644 --- a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py @@ -7,11 +7,12 @@ from unittest.mock import MagicMock, call, patch import pytest +from source_google_drive.spec import ServiceAccountCredentials, SourceGoogleDriveSpec +from source_google_drive.stream_reader import GoogleDriveRemoteFile, SourceGoogleDriveStreamReader + from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode -from source_google_drive.spec import ServiceAccountCredentials, SourceGoogleDriveSpec -from source_google_drive.stream_reader import GoogleDriveRemoteFile, SourceGoogleDriveStreamReader def create_reader( @@ -19,7 +20,7 @@ def create_reader( folder_url="https://drive.google.com/drive/folders/1Z2Q3", streams=[FileBasedStreamConfig(name="test", format=JsonlFormat())], credentials=ServiceAccountCredentials(auth_type="Service", service_account_info='{"test": "abc"}'), - ) + ), ): reader = SourceGoogleDriveStreamReader() reader.config = config diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py index 8dcb7e52e223..2740a3d88ee5 100644 --- a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py @@ -18,11 +18,11 @@ ("https://drive.google.com/drive/my-drive", None, True), ("http://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", None, True), ("https://drive.google.com/", None, True), - ] + ], ) def test_get_folder_id(input, output, raises): if raises: with pytest.raises(ValueError): get_folder_id(input) else: - assert get_folder_id(input) == output \ No newline at end of file + assert get_folder_id(input) == output diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py b/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py index 944377055e87..20d8f3b0132a 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py @@ -4,6 +4,7 @@ import json + # Check https://developers.google.com/webmaster-tools/search-console-api-original/v3/ for all available scopes OAUTH_SCOPE = "https://www.googleapis.com/auth/webmasters.readonly" diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py b/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py index 675661179530..6ceb6ae845a5 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py @@ -8,6 +8,7 @@ import requests + GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token" with open("credentials.json", "r") as f: diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py index 4e39115533b4..906add1a3e0e 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "requests", ] diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-search-console/main.py b/airbyte-integrations/connectors/source-google-search-console/main.py index 845383457bb7..62b40b4e0391 100755 --- a/airbyte-integrations/connectors/source-google-search-console/main.py +++ b/airbyte-integrations/connectors/source-google-search-console/main.py @@ -4,5 +4,6 @@ from source_google_search_console.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py index 2774f64703c7..7e0886377db3 100644 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py index 15759c39bec7..0530faa64a01 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py @@ -6,8 +6,10 @@ from google.auth.transport.requests import Request from google.oauth2.service_account import Credentials from requests.auth import AuthBase + from source_google_search_console.exceptions import UnauthorizedServiceAccountError + DEFAULT_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"] diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index e1481d1dcaab..c8c196c22f4c 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -10,6 +10,7 @@ import jsonschema import pendulum import requests + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -40,6 +41,7 @@ Sites, ) + custom_reports_schema = { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 1474a7591f7a..76d359743d65 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -9,10 +9,12 @@ import pendulum import requests +from requests.auth import AuthBase + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream -from requests.auth import AuthBase + BASE_URL = "https://www.googleapis.com/webmasters/v3/" ROW_LIMIT = 25000 diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py index 1a321480fc1f..63498cf5e2f4 100644 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_search_console.config_migrations import MigrateCustomReports from source_google_search_console.source import SourceGoogleSearchConsole +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 3c2b6dedfbe5..72acd67241e4 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -8,8 +8,6 @@ import pytest import requests -from airbyte_cdk.models import AirbyteConnectionStatus, Status, SyncMode -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest_lazyfixture import lazy_fixture from source_google_search_console.source import SourceGoogleSearchConsole from source_google_search_console.streams import ( @@ -23,6 +21,10 @@ ) from utils import command_check +from airbyte_cdk.models import AirbyteConnectionStatus, Status, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + logger = logging.getLogger("airbyte") @@ -133,7 +135,9 @@ def test_forbidden_should_retry(requests_mock, forbidden_error_message_json): def test_bad_aggregation_type_should_retry(requests_mock, bad_aggregation_type): stream = SearchAnalyticsKeywordSiteReportBySite(None, ["https://example.com"], "2021-01-01", "2021-01-02") - requests_mock.post(f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]}) + requests_mock.post( + f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]} + ) slice = list(stream.stream_slices(None))[0] url = stream.url_base + stream.path(None, slice) requests_mock.get(url, status_code=400, json=bad_aggregation_type) diff --git a/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-sheets/main.py b/airbyte-integrations/connectors/source-google-sheets/main.py index 806ac60fbefe..7ecf2f8cd0e9 100644 --- a/airbyte-integrations/connectors/source-google-sheets/main.py +++ b/airbyte-integrations/connectors/source-google-sheets/main.py @@ -4,5 +4,6 @@ from source_google_sheets.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py index 4abb69cece9e..1664da763bd0 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py @@ -11,6 +11,7 @@ from .helpers import SCOPES, Helpers + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py index 13b7d6f9b4bc..74da7aef6182 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py @@ -9,14 +9,16 @@ from datetime import datetime from typing import Dict, FrozenSet, Iterable, List, Tuple -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode from google.oauth2 import credentials as client_account from google.oauth2 import service_account from googleapiclient import discovery +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode + from .models.spreadsheet import RowData, Spreadsheet from .utils import safe_name_conversion + SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly", "https://www.googleapis.com/auth/drive.readonly"] logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py index eae6e3776123..06ba0e617d12 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py @@ -8,6 +8,10 @@ import socket from typing import Any, Generator, List, Mapping, MutableMapping, Optional, Union +from apiclient import errors +from google.auth import exceptions as google_exceptions +from requests.status_codes import codes as status_codes + from airbyte_cdk.models import FailureType from airbyte_cdk.models.airbyte_protocol import ( AirbyteCatalog, @@ -24,9 +28,6 @@ from airbyte_cdk.sources.streams.checkpoint import FullRefreshCheckpointReader from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.stream_status_utils import as_airbyte_message -from apiclient import errors -from google.auth import exceptions as google_exceptions -from requests.status_codes import codes as status_codes from .client import GoogleSheetsClient from .helpers import Helpers @@ -34,6 +35,7 @@ from .models.spreadsheet_values import SpreadsheetValues from .utils import exception_description_by_status_code, safe_name_conversion + # override default socket timeout to be 10 mins instead of 60 sec. # on behalf of https://github.com/airbytehq/oncall/issues/242 DEFAULT_SOCKET_TIMEOUT: int = 600 diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py index 0e2168c4aff2..bf3f6a8cc302 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py @@ -8,6 +8,7 @@ import unidecode from requests.status_codes import codes as status_codes + TOKEN_PATTERN = re.compile(r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P[^a-zA-Z\d]+)") DEFAULT_SEPARATOR = "_" diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py index 7f1a4c699a8b..6c8e392764df 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet, SpreadsheetValues, ValueRange +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.fixture def invalid_config(): @@ -28,10 +29,11 @@ def maker(spreadsheet_id, sheet_name): sheets=[ Sheet( data=[GridData(rowData=[RowData(values=[CellData(formattedValue="ID")])])], - properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}) + properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}), ), ], ) + return maker @@ -39,6 +41,7 @@ def maker(spreadsheet_id, sheet_name): def spreadsheet_values(): def maker(spreadsheet_id): return SpreadsheetValues(spreadsheetId=spreadsheet_id, valueRanges=[ValueRange(values=[["1"]])]) + return maker @@ -51,4 +54,5 @@ def maker(*name_schema_pairs): sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) + return maker diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py index d86d9fbaaa2b..f54e8cffb53a 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py @@ -6,9 +6,10 @@ from unittest.mock import patch from urllib.parse import parse_qsl, unquote, urlencode, urlunparse +from httplib2 import Response + from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.request import HttpRequest -from httplib2 import Response def parse_and_transform(parse_result_str: str): @@ -20,14 +21,16 @@ def parse_and_transform(parse_result_str: str): # Convert the ParseResult string into a dictionary components = eval(f"dict({parse_result_part})") - url = urlunparse(( - components["scheme"], - components["netloc"], - components["path"], - components["params"], - components["query"], - components["fragment"], - )) + url = urlunparse( + ( + components["scheme"], + components["netloc"], + components["path"], + components["params"], + components["query"], + components["fragment"], + ) + ) return url @@ -40,6 +43,7 @@ class CustomHttpMocker: Note: there is only support for get and post method and url matching ignoring the body but this is enough for the current test set. """ + requests_mapper: Dict = {} def post(self, request: HttpRequest, response: HttpResponse): @@ -62,9 +66,9 @@ def mock_request(self, uri, method="GET", body=None, headers=None, **kwargs): return mocked_response # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` - def __call__(self, test_func): # type: ignore + def __call__(self, test_func): # type: ignore @wraps(test_func) - def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed + def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed kwargs["http_mocker"] = self with patch("httplib2.Http.request", side_effect=self.mock_request): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py index af585b9e5fd4..9fc77d47a852 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py @@ -4,17 +4,18 @@ from airbyte_cdk.test.mock_http.request import HttpRequest + # todo: this should be picked from manifest in the future GOOGLE_SHEETS_BASE_URL = "https://sheets.googleapis.com/v4/spreadsheets" OAUTH_AUTHORIZATION_ENDPOINT = "https://oauth2.googleapis.com" + class RequestBuilder: @classmethod def get_account_endpoint(cls) -> RequestBuilder: return cls(resource="values:batchGet") - - def __init__(self, resource:str=None) -> None: + def __init__(self, resource: str = None) -> None: self._spreadsheet_id = None self._query_params = {} self._body = None diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py index 4fdcd7fe0309..e4781f4d267d 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import json + # this key was generated with rsa library from cryptography test_private_key = """ -----BEGIN PRIVATE KEY----- @@ -59,5 +60,5 @@ "auth_type": "Client", "client_id": "43987534895734985.apps.googleusercontent.com", "client_secret": "2347586435987643598", - "refresh_token": "1//4398574389537495437983457985437" - } + "refresh_token": "1//4398574389537495437983457985437", +} diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py index 6075f6853592..2123a2caec3b 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py @@ -5,10 +5,14 @@ from copy import deepcopy from typing import Optional from unittest import TestCase -from unittest.mock import patch # patch("time.sleep") -from unittest.mock import Mock +from unittest.mock import ( + Mock, + patch, # patch("time.sleep") +) import pytest +from source_google_sheets import SourceGoogleSheets + from airbyte_cdk.models import Status from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder @@ -16,12 +20,12 @@ from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.utils import AirbyteTracedException -from source_google_sheets import SourceGoogleSheets from .custom_http_mocker import CustomHttpMocker as HttpMocker from .request_builder import AuthBuilder, RequestBuilder from .test_credentials import oauth_credentials, service_account_credentials, service_account_info + _SPREADSHEET_ID = "a_spreadsheet_id" _STREAM_NAME = "a_stream_name" @@ -29,15 +33,9 @@ _C_STREAM_NAME = "c_stream_name" -_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": oauth_credentials -} +_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": oauth_credentials} -_SERVICE_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": service_account_credentials -} +_SERVICE_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": service_account_credentials} class GoogleSheetSourceTest(TestCase): @@ -49,14 +47,11 @@ def setUp(self) -> None: @staticmethod def authorize(http_mocker: HttpMocker): # Authorization request with user credentials to "https://oauth2.googleapis.com" to obtain a token - http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_response", __file__)), 200) - ) + http_mocker.post(AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_response", __file__)), 200)) @staticmethod - def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=None, meta_response_code: Optional[int]=200): - """" + def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str] = None, meta_response_code: Optional[int] = 200): + """ " Mock request to https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=false&alt=json in order to obtain sheets (streams) from the spreed_sheet_id provided. e.g. from response file @@ -79,12 +74,19 @@ def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=No if streams_response_file: http_mocker.get( RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(False).with_alt("json").build(), - HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code) + HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code), ) @staticmethod - def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_response_code: int=200, stream_name: Optional[str]=_STREAM_NAME, data_initial_range_response_file: Optional[str]=None, data_initial_response_code: Optional[int]=200): - """" + def get_schema( + http_mocker: HttpMocker, + headers_response_file: str, + headers_response_code: int = 200, + stream_name: Optional[str] = _STREAM_NAME, + data_initial_range_response_file: Optional[str] = None, + data_initial_response_code: Optional[int] = 200, + ): + """ " Mock request to 'https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=true&ranges=!1:1&alt=json' to obtain headers data (keys) used for stream schema from the spreadsheet + sheet provided. For this we use range of first row in query. @@ -127,13 +129,20 @@ def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_resp ]}],}]}] """ http_mocker.get( - RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(True).with_ranges(f"{stream_name}!1:1").with_alt("json").build(), - HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code) + RequestBuilder() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_include_grid_data(True) + .with_ranges(f"{stream_name}!1:1") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code), ) @staticmethod - def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int=200, stream_name:Optional[str]=_STREAM_NAME): - """" + def get_stream_data( + http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int = 200, stream_name: Optional[str] = _STREAM_NAME + ): + """ " Mock requests to 'https://sheets.googleapis.com/v4/spreadsheets//values:batchGet?ranges=!2:202&majorDimension=ROWS&alt=json' to obtain value ranges (data) for stream from the spreadsheet + sheet provided. For this we use range [2:202(2 + range in config which default is 200)]. @@ -156,8 +165,13 @@ def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, rang """ batch_request_ranges = f"{stream_name}!2:202" http_mocker.get( - RequestBuilder.get_account_endpoint().with_spreadsheet_id(_SPREADSHEET_ID).with_ranges(batch_request_ranges).with_major_dimension("ROWS").with_alt("json").build(), - HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code) + RequestBuilder.get_account_endpoint() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_ranges(batch_request_ranges) + .with_major_dimension("ROWS") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code), ) @HttpMocker() @@ -186,15 +200,12 @@ def test_given_service_authentication_error_when_check_then_status_is_failed(sel @HttpMocker() def test_invalid_credentials_error_message_when_check(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." - ) + assert str(exc_info.value) == ("Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.") @HttpMocker() def test_invalid_link_error_message_when_check(self, http_mocker: HttpMocker) -> None: @@ -216,9 +227,7 @@ def test_check_access_expired(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_permissions", 403) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Config error: " - ) + assert str(exc_info.value) == ("Config error: ") @HttpMocker() def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) -> None: @@ -227,7 +236,10 @@ def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f'Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. ' + assert ( + connection_status.message + == f"Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " + ) @HttpMocker() def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: @@ -236,7 +248,10 @@ def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + assert ( + connection_status.message + == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + ) @HttpMocker() def test_given_grid_sheet_type_with_at_least_one_row_when_discover_then_return_stream(self, http_mocker: HttpMocker) -> None: @@ -252,9 +267,9 @@ def test_discover_return_expected_schema(self, http_mocker: HttpMocker) -> None: # When we move to manifest only is possible that DynamicSchemaLoader will identify fields like age as integers # and addresses "oneOf": [{"type": ["null", "string"]}, {"type": ["null", "integer"]}] as it has mixed data expected_schemas_properties = { - _STREAM_NAME: {'age': {'type': 'string'}, 'name': {'type': 'string'}}, - _B_STREAM_NAME: {'email': {'type': 'string'}, 'name': {'type': 'string'}}, - _C_STREAM_NAME: {'address': {'type': 'string'}} + _STREAM_NAME: {"age": {"type": "string"}, "name": {"type": "string"}}, + _B_STREAM_NAME: {"email": {"type": "string"}, "name": {"type": "string"}}, + _C_STREAM_NAME: {"address": {"type": "string"}}, } GoogleSheetSourceTest.get_streams(http_mocker, "multiple_streams_schemas_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, f"multiple_streams_schemas_{_STREAM_NAME}_range", 200) @@ -293,13 +308,12 @@ def test_discover_could_not_run_discover(self, http_mocker: HttpMocker) -> None: @HttpMocker() def test_discover_invalid_credentials_error_message(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(Exception) as exc_info: self._source.discover(Mock(), self._config) - expected_message = 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + expected_message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." assert str(exc_info.value) == expected_message @HttpMocker() @@ -307,7 +321,6 @@ def test_discover_404_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_link", 404) with pytest.raises(AirbyteTracedException) as exc_info: - self._source.discover(Mock(), self._config) expected_message = ( f"The requested Google Sheets spreadsheet with id {_SPREADSHEET_ID} does not exist." @@ -320,7 +333,6 @@ def test_discover_403_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_permissions", 403) with pytest.raises(AirbyteTracedException) as exc_info: - self._source.discover(Mock(), self._config) expected_message = ( "The authenticated Google Sheets user does not have permissions to view the " @@ -350,7 +362,15 @@ def test_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range") GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 2 @@ -367,20 +387,23 @@ def test_when_read_multiple_streams_return_records(self, http_mocker: HttpMocker GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_B_STREAM_NAME}_range_2", stream_name=_B_STREAM_NAME) GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_C_STREAM_NAME}_range_2", stream_name=_C_STREAM_NAME) - configured_catalog = (CatalogBuilder().with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME). - with_json_schema({"properties": {'age': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_B_STREAM_NAME). - with_json_schema({"properties": {'email': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME). - with_json_schema({"properties": {'address': {'type': 'string'}} - }) + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"age": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_B_STREAM_NAME) + .with_json_schema({"properties": {"email": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME).with_json_schema({"properties": {"address": {"type": "string"}}}) + ) + .build() ) - .build()) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 9 @@ -403,7 +426,15 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_2", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions_2") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 5 @@ -414,21 +445,26 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt assert output.trace_messages[1].trace.stream_status.status == AirbyteStreamStatus.RUNNING assert output.trace_messages[2].trace.stream_status.status == AirbyteStreamStatus.COMPLETE - @HttpMocker() def test_read_429_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "rate_limit_error", 429) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -437,13 +473,19 @@ def test_read_403_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "invalid_permissions", 403) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" - ) + expected_message = f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -452,14 +494,20 @@ def test_read_500_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "internal_server_error", 500) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" - ) + expected_message = "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -467,12 +515,18 @@ def test_read_empty_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_empty", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message @HttpMocker() @@ -480,10 +534,16 @@ def test_read_expected_data_on_1_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_with_unexpected_extra_sheet", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index 5a1408faa0c1..a57d024c6d0c 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -7,6 +7,10 @@ import unittest from unittest.mock import Mock, patch +from source_google_sheets.client import GoogleSheetsClient +from source_google_sheets.helpers import Helpers +from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet + from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, AirbyteStream, @@ -15,9 +19,7 @@ DestinationSyncMode, SyncMode, ) -from source_google_sheets.client import GoogleSheetsClient -from source_google_sheets.helpers import Helpers -from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index b939910d57f7..54a2e054d91e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -6,14 +6,15 @@ import pytest import requests -from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog -from airbyte_cdk.utils import AirbyteTracedException from apiclient import errors from source_google_sheets import SourceGoogleSheets from source_google_sheets.client import GoogleSheetsClient from source_google_sheets.helpers import SCOPES, Helpers from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet +from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog +from airbyte_cdk.utils import AirbyteTracedException + def set_http_error_for_google_sheets_client(mocker, resp): mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) @@ -191,7 +192,7 @@ def test_discover_invalid_credentials_error_message(mocker, invalid_config): source = SourceGoogleSheets() with pytest.raises(AirbyteTracedException) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - assert e.value.args[0] == 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + assert e.value.args[0] == "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." def test_get_credentials(invalid_config): @@ -223,12 +224,14 @@ def test_read_429_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) - expected_message = ( - "Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Rate limit has been reached. Please try later or request a higher quota for your account." assert e.value.args[0] == expected_message @@ -243,7 +246,11 @@ def test_read_403_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) assert ( @@ -265,7 +272,11 @@ def test_read_500_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) expected_message = ( @@ -303,8 +314,13 @@ def test_read_empty_sheet(invalid_config, mocker, catalog, caplog): sheet1 = "soccer_team" sheet2 = "soccer_team2" sheets = [ - Sheet(properties=SheetProperties(title=t), data=[{"test1": "12", "test2": "123"},]) - for t in [sheet1] + Sheet( + properties=SheetProperties(title=t), + data=[ + {"test1": "12", "test2": "123"}, + ], + ) + for t in [sheet1] ] mocker.patch.object( GoogleSheetsClient, @@ -328,7 +344,11 @@ def test_when_read_then_status_messages_emitted(mocker, spreadsheet, spreadsheet mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed @@ -346,7 +366,11 @@ def test_when_read_then_state_message_emitted(mocker, spreadsheet, spreadsheet_v mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-greenhouse/main.py b/airbyte-integrations/connectors/source-greenhouse/main.py index e08a14b429fd..378686d959b4 100644 --- a/airbyte-integrations/connectors/source-greenhouse/main.py +++ b/airbyte-integrations/connectors/source-greenhouse/main.py @@ -4,5 +4,6 @@ from source_greenhouse.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py index dabad443b366..2be3db68fbff 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py index 605c45c1ea2f..463a456b4aaa 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py @@ -5,9 +5,10 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.sources.streams import Stream from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer +from airbyte_cdk.sources.streams import Stream + @pytest.fixture def greenhouse_slicer(): @@ -18,4 +19,11 @@ def greenhouse_slicer(): @pytest.fixture def greenhouse_substream_slicer(): parent_stream = MagicMock(spec=Stream) - return GreenHouseSubstreamSlicer(cursor_field='cursor_field', stream_slice_field='slice_field', parent_stream=parent_stream, parent_key='parent_key', parameters={}, request_cursor_field=None) + return GreenHouseSubstreamSlicer( + cursor_field="cursor_field", + stream_slice_field="slice_field", + parent_stream=parent_stream, + parent_key="parent_key", + parameters={}, + request_cursor_field=None, + ) diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py index 48db265f477f..27bab35ef71e 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -6,9 +6,10 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.sources.streams import Stream from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer +from airbyte_cdk.sources.streams import Stream + def test_slicer(greenhouse_slicer): date_time = "2022-09-05T10:10:10.000000Z" @@ -53,17 +54,12 @@ def test_sub_slicer(last_record, expected, records): @pytest.mark.parametrize( "stream_state, cursor_field, expected_state", [ - ({'cursor_field_1': '2022-09-05T10:10:10.000Z'}, 'cursor_field_1', {'cursor_field_1': '2022-09-05T10:10:10.000Z'}), - ({'cursor_field_2': '2022-09-05T10:10:100000Z'}, 'cursor_field_3', {}), - ({'cursor_field_4': None}, 'cursor_field_4', {}), - ({'cursor_field_5': ''}, 'cursor_field_5', {}), + ({"cursor_field_1": "2022-09-05T10:10:10.000Z"}, "cursor_field_1", {"cursor_field_1": "2022-09-05T10:10:10.000Z"}), + ({"cursor_field_2": "2022-09-05T10:10:100000Z"}, "cursor_field_3", {}), + ({"cursor_field_4": None}, "cursor_field_4", {}), + ({"cursor_field_5": ""}, "cursor_field_5", {}), ], - ids=[ - "cursor_value_present", - "cursor_value_not_present", - "cursor_value_is_None", - "cursor_value_is_empty_string" - ] + ids=["cursor_value_present", "cursor_value_not_present", "cursor_value_is_None", "cursor_value_is_empty_string"], ) def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer = GreenHouseSlicer(cursor_field=cursor_field, parameters={}, request_cursor_field=None) @@ -71,36 +67,30 @@ def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer.set_initial_state(stream_state) assert slicer.get_stream_state() == expected_state + @pytest.mark.parametrize( "stream_state, initial_state, expected_state", [ ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'}}, - { - 'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}, - 'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'} - } + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}, "id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, ), ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T09:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}} - ), - ( - {}, - {}, - {} + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T09:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, ), + ({}, {}, {}), ], ids=[ "stream_state and initial_state have different keys", "stream_state and initial_state have overlapping keys with different values", - "stream_state and initial_state are empty" - ] + "stream_state and initial_state are empty", + ], ) def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, initial_state, expected_state): - slicer = greenhouse_substream_slicer + slicer = greenhouse_substream_slicer # Set initial state slicer._state = initial_state slicer.set_initial_state(stream_state) @@ -110,27 +100,11 @@ def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, @pytest.mark.parametrize( "first_record, second_record, expected_result", [ - ( - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ( - {'cursor_field': '2023-02-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - True - ), - ( - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - {'cursor_field': ''}, - True - ), - ( - {'cursor_field': ''}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ] + ({"cursor_field": "2023-01-01T00:00:00.000Z"}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ({"cursor_field": "2023-02-01T00:00:00.000Z"}, {"cursor_field": "2023-01-01T00:00:00.000Z"}, True), + ({"cursor_field": "2023-01-02T00:00:00.000Z"}, {"cursor_field": ""}, True), + ({"cursor_field": ""}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ], ) def test_is_greater_than_or_equal(greenhouse_substream_slicer, first_record, second_record, expected_result): slicer = greenhouse_substream_slicer diff --git a/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gridly/main.py b/airbyte-integrations/connectors/source-gridly/main.py index 307be6500faf..48b578302a7b 100644 --- a/airbyte-integrations/connectors/source-gridly/main.py +++ b/airbyte-integrations/connectors/source-gridly/main.py @@ -4,5 +4,6 @@ from source_gridly.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py b/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py index 05f9d4843cc5..b896e6102f94 100644 --- a/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py @@ -5,6 +5,7 @@ from typing import Any, Dict import requests + from airbyte_cdk.models import AirbyteStream from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/source.py b/airbyte-integrations/connectors/source-gridly/source_gridly/source.py index 10a400672cfb..d1c2ed8a5483 100644 --- a/airbyte-integrations/connectors/source-gridly/source_gridly/source.py +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/source.py @@ -9,6 +9,7 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.models import AirbyteCatalog from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py index 2aa8d22ce2d9..4d3ea285f4f4 100644 --- a/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py @@ -6,6 +6,7 @@ from source_gridly.source import SourceGridly + CONFIG = {"api_key": "IbuIBdkFjrJps6", "grid_id": "4539o52kmdjmzwp"} diff --git a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hardcoded-records/main.py b/airbyte-integrations/connectors/source-hardcoded-records/main.py index ead17772f9c3..e697227784c1 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/main.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/main.py @@ -5,5 +5,6 @@ from source_hardcoded_records.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py b/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py index 862ac9470e96..73b2369c5792 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py @@ -9,6 +9,7 @@ from .streams import Customers, DummyFields, Products + DEFAULT_COUNT = 1_000 diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py index a33c09315382..237fb7b71ce1 100644 --- a/airbyte-integrations/connectors/source-harness/main.py +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -4,5 +4,6 @@ from source_harness.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-harness/source_harness/run.py b/airbyte-integrations/connectors/source-harness/source_harness/run.py index 544daa9407a1..6a0dc730fd62 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/run.py +++ b/airbyte-integrations/connectors/source-harness/source_harness/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_harness import SourceHarness +from airbyte_cdk.entrypoint import launch + def run(): source = SourceHarness() diff --git a/airbyte-integrations/connectors/source-harness/source_harness/source.py b/airbyte-integrations/connectors/source-harness/source_harness/source.py index a52f4c06db86..b12559ebc98a 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/source.py +++ b/airbyte-integrations/connectors/source-harness/source_harness/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py index 9e7e8054a273..05c6d7814809 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py @@ -5,9 +5,10 @@ import logging import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type from source_hubspot.source import SourceHubspot +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type + @pytest.fixture def source(): diff --git a/airbyte-integrations/connectors/source-hubspot/main.py b/airbyte-integrations/connectors/source-hubspot/main.py index dc073ca21ed6..8aa42945418c 100644 --- a/airbyte-integrations/connectors/source-hubspot/main.py +++ b/airbyte-integrations/connectors/source-hubspot/main.py @@ -4,5 +4,6 @@ from source_hubspot.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py index e73313f80e3f..925f2c84f06f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py @@ -6,9 +6,10 @@ from typing import Any import requests +from requests import HTTPError + from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError class HubspotError(AirbyteTracedException): diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 7bc5eda8ce53..195ac0335d21 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -8,12 +8,13 @@ from itertools import chain from typing import Any, Generator, List, Mapping, Optional, Tuple, Union +from requests import HTTPError + from airbyte_cdk.models import FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction -from requests import HTTPError from source_hubspot.errors import HubspotInvalidAuth from source_hubspot.streams import ( API, @@ -68,6 +69,7 @@ Workflows, ) + """ https://github.com/airbytehq/oncall/issues/3800 we use start date 2006-01-01 as date of creation of Hubspot to retrieve all data if start date was not provided diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 2cbfbdd6a2c9..4daedbd18fcb 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -15,6 +15,8 @@ import backoff import pendulum as pendulum import requests +from requests import HTTPError, codes + from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import Source @@ -29,7 +31,6 @@ from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError, codes from source_hubspot.components import NewtoLegacyFieldTransformation from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout, InvalidStartDateConfigError @@ -44,6 +45,7 @@ StoreAsIs, ) + # we got this when provided API Token has incorrect format CLOUDFLARE_ORIGIN_DNS_ERROR = 530 @@ -1515,14 +1517,12 @@ def cursor_field_datetime_format(self) -> str: class ContactsFormSubmissions(ContactsAllBase, ResumableFullRefreshMixin, ABC): - records_field = "form-submissions" filter_field = "formSubmissionMode" filter_value = "all" class ContactsMergedAudit(ContactsAllBase, ResumableFullRefreshMixin, ABC): - records_field = "merge-audits" unnest_fields = ["merged_from_email", "merged_to_email"] @@ -2137,7 +2137,6 @@ def _transform(self, records: Iterable) -> Iterable: class CompaniesPropertyHistory(PropertyHistoryV3): - scopes = {"crm.objects.companies.read"} properties_scopes = {"crm.schemas.companies.read"} entity = "companies" @@ -2419,7 +2418,6 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: now = pendulum.now(tz="UTC") for parent_slice in super().stream_slices(sync_mode, cursor_field, stream_state): - object_id = parent_slice["parent"][self.object_id_field] # We require this workaround to shorten the duration of the acceptance test run. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index deecb0b377d3..8d10c01fde86 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -6,6 +6,7 @@ from source_hubspot.source import SourceHubspot from source_hubspot.streams import API + NUMBER_OF_PROPERTIES = 2000 @@ -81,9 +82,16 @@ def some_credentials_fixture(): def fake_properties_list(): return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] + @pytest.fixture(name="migrated_properties_list") def migrated_properties_list(): - return ["hs_v2_date_entered_prospect", "hs_v2_date_exited_prospect", "hs_v2_cumulative_time_in_prsopect", "hs_v2_some_other_property_in_prospect"] + return [ + "hs_v2_date_entered_prospect", + "hs_v2_date_exited_prospect", + "hs_v2_cumulative_time_in_prsopect", + "hs_v2_some_other_property_in_prospect", + ] + @pytest.fixture(name="api") def api(some_credentials): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py index 048142759ca2..1927b02a1705 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py @@ -5,9 +5,7 @@ class ConfigBuilder: def __init__(self): - self._config = { - "enable_experimental_streams": True - } + self._config = {"enable_experimental_streams": True} def with_start_date(self, start_date: str): self._config["start_date"] = start_date diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py index 17ba71bebf3c..e9695115f5a3 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py @@ -26,8 +26,7 @@ def with_refresh_token(self, refresh_token: str): def build(self) -> HttpRequest: client_id, client_secret, refresh_token = self._params["client_id"], self._params["client_secret"], self._params["refresh_token"] return HttpRequest( - url=self.URL, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + url=self.URL, body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" ) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index d0258a26500a..614c51e8ece6 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -29,12 +29,7 @@ def with_query(self, qp): return self def build(self) -> HttpRequest: - return HttpRequest( - url=self.URL, - query_params=self._query_params, - headers=self.headers, - body=self._request_body - ) + return HttpRequest(url=self.URL, query_params=self._query_params, headers=self.headers, body=self._request_body) class CRMStreamRequestBuilder(AbstractRequestBuilder): @@ -78,14 +73,7 @@ def _archived(self): @property def _query_params(self): - return [ - self._archived, - self._associations, - self._limit, - self._after, - self._dt_range, - self._properties - ] + return [self._archived, self._associations, self._limit, self._after, self._dt_range, self._properties] def build(self): q = "&".join(filter(None, self._query_params)) @@ -96,14 +84,7 @@ def build(self): class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): @property def _query_params(self): - return [ - self._limit, - self._after, - self._dt_range, - self._archived, - self._associations, - self._properties - ] + return [self._limit, self._after, self._dt_range, self._archived, self._associations, self._properties] class OwnersArchivedStreamRequestBuilder(AbstractRequestBuilder): @@ -122,11 +103,14 @@ def _archived(self): @property def _query_params(self): - return filter(None, [ - self._limit, - self._after, - self._archived, - ]) + return filter( + None, + [ + self._limit, + self._after, + self._archived, + ], + ) def with_page_token(self, next_page_token: Dict): self._after = "&".join([f"{str(key)}={str(val)}" for key, val in next_page_token.items()]) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py index 87ba98af2b5a..b5c2cc79e61b 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py @@ -7,13 +7,13 @@ from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template + _CONTACTS_FIELD = "contacts" _FORM_SUBMISSIONS_FIELD = "form-submissions" _LIST_MEMBERSHIPS_FIELD = "list-memberships" _MERGE_AUDITS_FIELD = "merge-audits" - def _get_template() -> Dict[str, Any]: return find_template("all_contacts", __file__) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py index 595a02232d43..f5bb912822f7 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -12,7 +12,7 @@ def __init__( self, template: List[Any], records_path: Optional[Union[FieldPath, NestedPath]] = None, - pagination_strategy: Optional[PaginationStrategy] = None + pagination_strategy: Optional[PaginationStrategy] = None, ): self._response = template self._records: List[RecordBuilder] = [] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index ded25153204f..7f594d984fdf 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -9,13 +9,4 @@ class HubspotPaginationStrategy(PaginationStrategy): NEXT_PAGE_TOKEN = {"after": "256"} def update(self, response: Dict[str, Any]) -> None: - response["paging"] = { - "next": { - "link": "link_to_the_next_page", - **self.NEXT_PAGE_TOKEN - }, - "prev": { - "before": None, - "link": None - } - } + response["paging"] = {"next": {"link": "link_to_the_next_page", **self.NEXT_PAGE_TOKEN}, "prev": {"before": None, "link": None}} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index f5d4795c3775..6b02d952e14c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -13,7 +13,7 @@ class HubspotStreamResponseBuilder(HttpResponseBuilder): @property def pagination_strategy(self): return self._pagination_strategy - + @classmethod def for_stream(cls, stream: str): return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py index 7bcddfe5e179..fdf05847617c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py @@ -3,6 +3,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -10,6 +11,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsFormSubmissionsBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _VID_OFFSET = 5331889818 @@ -32,43 +34,60 @@ def tearDown(self) -> None: def test_read_multiple_contact_pages(self) -> None: first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( - cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), - stream=self.STREAM_NAME, - sync_mode=SyncMode.full_refresh + cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh ) self._http_mocker.assert_number_of_calls(first_page_request, 2) @@ -87,56 +106,73 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py index 8fb6e598a395..3c21d185a650 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py @@ -4,6 +4,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import SyncMode @@ -12,6 +13,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsListMembershipBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _NOW = datetime.now(timezone.utc) @@ -37,25 +39,40 @@ def test_given_pagination_when_read_then_extract_records_from_both_pages(self) - self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).with_vid_offset(str(_VID_OFFSET)).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ]), - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ] + ), + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -67,14 +84,22 @@ def test_given_timestamp_before_start_date_when_read_then_filter_out(self) -> No self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), + ] + ), + ] + ) + .build(), + ) + output = self.read_from_stream( + self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh ) - output = self.read_from_stream(self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -83,12 +108,18 @@ def test_given_state_when_read_then_filter_out(self) -> None: self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py index 18c67317aebf..8b76e13cc6c4 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py @@ -3,6 +3,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -10,6 +11,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsMergeAuditsBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _VID_OFFSET = 5331889818 @@ -36,32 +38,49 @@ def test_read_multiple_contact_pages(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -82,9 +101,8 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] @@ -94,39 +112,56 @@ def test_read_from_incoming_state(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index d37005531ecf..936f68a4fa2a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -5,6 +5,7 @@ import freezegun import mock + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -27,22 +28,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +82,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +99,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,8 +136,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 - diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py index d07dbd20985a..5f46b8982ae8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py @@ -5,6 +5,7 @@ import freezegun import mock + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -27,22 +28,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +82,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +99,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,7 +136,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index d54a94ca6a9d..eef01cbc0fc5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -16,6 +17,7 @@ class TestOwnersArchivedStream(HubspotTestCase): The test case contains a single test - this is just a sanity check, as the tested stream is identical to the `Owners` stream (which is covered by acceptance tests), except for a single url param. """ + SCOPES = ["crm.objects.owners.read"] CURSOR_FIELD = "updatedAt" STREAM_NAME = "owners_archived" @@ -28,10 +30,10 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -54,7 +56,7 @@ def test_given_two_pages_when_read_stream_private_token_then_return_records(self self.mock_response( http_mocker, self.request().with_page_token(self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN).build(), - self.response().build() + self.response().build(), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 7e3889a69ee3..bd22328de3ec 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -8,6 +8,7 @@ import mock import pytest import pytz + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -16,6 +17,7 @@ from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder from .response_builder.streams import HubspotStreamResponseBuilder + CRM_STREAMS = ( ("tickets_web_analytics", "tickets", "ticket", ["contacts", "deals", "companies"]), ("deals_web_analytics", "deals", "deal", ["contacts", "companies", "line_items"]), @@ -51,17 +53,11 @@ def web_analytics_request( object_type: str, start_date: Optional[str] = None, end_date: Optional[str] = None, - first_page: bool = True + first_page: bool = True, ): start_date = start_date or cls.dt_str(cls.start_date()) end_date = end_date or cls.dt_str(cls.now()) - query = { - "limit": 100, - "occurredAfter": start_date, - "occurredBefore": end_date, - "objectId": object_id, - "objectType": object_type - } + query = {"limit": 100, "occurredAfter": start_date, "occurredBefore": end_date, "objectId": object_id, "objectType": object_type} if not first_page: query.update(cls.response_builder(stream).pagination_strategy.NEXT_PAGE_TOKEN) @@ -95,10 +91,10 @@ def mock_parent_object( ): response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: @@ -136,7 +132,7 @@ def test_given_one_page_when_read_stream_oauth_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.oauth_config(), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -154,7 +150,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -172,12 +168,12 @@ def test_given_two_pages_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, with_pagination=True) + self.web_analytics_response(stream_name, with_pagination=True), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type, first_page=False), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -196,7 +192,7 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, with_pagination=True, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_parent_object( http_mocker, @@ -205,17 +201,17 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, first_page=False, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, "another_object_id", object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -236,7 +232,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()), - date_range=start_to_end + date_range=start_to_end, ) for dt_range in date_ranges: for _id in (self.OBJECT_ID, "another_object_id"): @@ -245,7 +241,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, _id, object_type, start, end), - web_analytics_response + web_analytics_response, ) config_start_dt = date_ranges[0][0] output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN, config_start_dt), stream_name, SyncMode.full_refresh) @@ -264,7 +260,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=500, body="{}") + HttpResponse(status_code=500, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -285,10 +281,7 @@ def test_given_500_then_200_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - [ - HttpResponse(status_code=500, body="{}"), - self.web_analytics_response(stream_name) - ] + [HttpResponse(status_code=500, body="{}"), self.web_analytics_response(stream_name)], ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -299,12 +292,7 @@ def test_given_500_then_200_when_read_then_return_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_missing_scopes_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) @@ -313,12 +301,7 @@ def test_given_missing_scopes_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_unauthorized_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -328,7 +311,7 @@ def test_given_unauthorized_error_when_read_then_hault( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}") + HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -339,12 +322,7 @@ def test_given_unauthorized_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_transformed_records( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -354,7 +332,7 @@ def test_given_one_page_when_read_then_get_transformed_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) record = output.records[0].record.data @@ -365,12 +343,7 @@ def test_given_one_page_when_read_then_get_transformed_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_no_records_filtered( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): # validate that no filter is applied on the record set received from the API response self.mock_custom_objects(http_mocker) @@ -381,7 +354,7 @@ def test_given_one_page_when_read_then_get_no_records_filtered( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))) + self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -399,11 +372,9 @@ def test_given_incremental_sync_when_read_then_state_message_produced_and_state_ self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) - ) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental) assert len(output.state_messages) == 1 cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) @@ -423,15 +394,15 @@ def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) another_object_id = "another_object_id" current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}) - ) + stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -453,14 +424,14 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}) - ) + stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -493,24 +464,23 @@ def mock_parent_object( date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: response_builder = response_builder.with_pagination() start, end = date_range - request_builder = IncrementalCRMStreamRequestBuilder().for_entity( - object_type - ).with_associations( - associations - ).with_dt_range( - ("startTimestamp", cls.dt_conversion(start)), - ("endTimestamp", cls.dt_conversion(end)) - ).with_properties(properties) + request_builder = ( + IncrementalCRMStreamRequestBuilder() + .for_entity(object_type) + .with_associations(associations) + .with_dt_range(("startTimestamp", cls.dt_conversion(start)), ("endTimestamp", cls.dt_conversion(end))) + .with_properties(properties) + ) if not first_page: request_builder = request_builder.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) @@ -533,12 +503,8 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_two_pages_when_read_then_return_records( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_two_pages_when_read_then_return_records( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_two_pages_when_read_then_return_records(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_two_pages_when_read_then_return_records(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( @@ -573,12 +539,8 @@ def test_given_missing_scopes_error_when_read_then_hault( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_unauthorized_error_when_read_then_hault( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_unauthorized_error_when_read_then_hault( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_unauthorized_error_when_read_then_hault(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_unauthorized_error_when_read_then_hault(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_one_page_when_read_then_get_transformed_records( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py index b57f773270f9..e83609da66b8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py @@ -25,32 +25,20 @@ "hs_v2_date_exited_prospect": {"type": ["null", "string"]}, "hs_date_exited_prospect": {"type": ["null", "string"]}, "hs_v2_some_other_field": {"type": ["null", "string"]}, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "hs_v2_date_entered_poetry": 1827 - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827}, { "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "properties": { - "hs_v2_date_entered_poetry": 1827 - } - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "properties": {"hs_v2_date_entered_poetry": 1827}}, { "name": "Edgar Allen Poe", "age": 215, @@ -58,8 +46,8 @@ "properties": { "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } - } + }, + }, ), ( { @@ -71,19 +59,15 @@ "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", - } + }, ), ( - { - "name": "Edgar Allen Poe", - "hs_v2_date_entered_poetry": 1827, - "hs_date_entered_poetry": 9999 - }, + {"name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999}, { "name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999, - } + }, ), ], ids=[ @@ -91,7 +75,7 @@ "Transforms record w/ flat properties", "Transform record w/ nested properties", "Does not transform record w/o need to transformation", - "Does not overwrite value for legacy field if legacy field exists" + "Does not overwrite value for legacy field if legacy field exists", ], ) def test_new_to_legacy_field_transformation(input, expected): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py index a5793b49b957..454791f22202 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py @@ -37,7 +37,6 @@ def test_field_type_format_converting(field_type, expected): ], ) def test_bad_field_type_converting(field_type, expected, caplog, capsys): - assert Stream._get_field_props(field_type=field_type) == expected logs = caplog.records diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index b3c7e71e0d7f..2033d84ab433 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -12,14 +12,16 @@ import mock import pendulum import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type from source_hubspot.errors import HubspotRateLimited, InvalidStartDateConfigError from source_hubspot.helpers import APIv3Property from source_hubspot.source import SourceHubspot from source_hubspot.streams import API, Companies, Deals, Engagements, MarketingEmails, Products, Stream +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type + from .utils import read_full_refresh, read_incremental + NUMBER_OF_PROPERTIES = 2000 logger = logging.getLogger("test_client") @@ -83,7 +85,6 @@ def test_check_connection_invalid_start_date_exception(config_invalid_date): @mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams") def test_streams(requests_mock, config): - streams = SourceHubspot().streams(config) assert len(streams) == 35 @@ -91,7 +92,6 @@ def test_streams(requests_mock, config): @mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams") def test_streams_incremental(requests_mock, config_experimental): - streams = SourceHubspot().streams(config_experimental) assert len(streams) == 47 @@ -141,7 +141,7 @@ def test_cast_datetime(common_params, caplog): # if you find some diff locally try using "Ex: argument of type 'DateTime' is not iterable in the message". There could be a # difference in local environment when pendulum.parsing.__init__.py importing parse_iso8601. Anyway below is working fine # in container for now and I am not sure if this diff was just a problem with my setup. - "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object" + "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object", }, } assert expected_warning_message["log"]["message"] in caplog.text @@ -488,6 +488,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req assert len(records) == 11000 assert test_stream.state["updatedAt"] == test_stream._init_sync.to_iso8601_string() + def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common_params, fake_properties_list): """ If there are more than 10,000 records that would be returned by the Hubspot search endpoint, @@ -500,7 +501,7 @@ def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common test_stream.associations = [] def random_date(start, end): - return pendulum.from_timestamp(random.randint(start, end)/1000).to_iso8601_string() + return pendulum.from_timestamp(random.randint(start, end) / 1000).to_iso8601_string() after = 0 @@ -518,17 +519,13 @@ def custom_callback(request, context): id = int(filters[2].get("value", 0)) next = int(after) + 100 results = [ - { - "id": f"{y + id}", - "updatedAt": random_date(min_time, max_time), - "after": after - } for y in range(int(after) + 1, next + 1) + {"id": f"{y + id}", "updatedAt": random_date(min_time, max_time), "after": after} for y in range(int(after) + 1, next + 1) ] context.status_code = 200 - if ((id + next) < 11000): + if (id + next) < 11000: return {"results": results, "paging": {"next": {"after": f"{next}"}}} else: - return {"results": results, "paging": {}} # Last page + return {"results": results, "paging": {}} # Last page properties_response = [ { @@ -787,6 +784,7 @@ def test_get_granted_scopes(requests_mock, mocker): assert expected_scopes == actual_scopes + def test_get_granted_scopes_retry(requests_mock, mocker): authenticator = mocker.Mock() expected_token = "the-token" diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py index 86534f61540d..632ed4cf3554 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py @@ -5,6 +5,7 @@ import pytest from source_hubspot.helpers import APIv1Property, APIv3Property + lorem_ipsum = """Lorem ipsum dolor sit amet, consectetur adipiscing elit""" lorem_ipsum = lorem_ipsum.lower().replace(",", "") diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 23ba97a303e7..064f98512101 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -7,7 +7,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from source_hubspot.streams import ( Campaigns, Companies, @@ -44,6 +43,8 @@ Workflows, ) +from airbyte_cdk.models import SyncMode + from .utils import read_full_refresh, read_incremental @@ -169,9 +170,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para contact_lists_v1_response = [ { "json": { - "contacts": [ - {"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]} - ] + "contacts": [{"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]}] }, "status_code": 200, } @@ -193,6 +192,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para records = read_full_refresh(stream) assert records + @pytest.mark.parametrize( "stream, endpoint, cursor_value", [ @@ -203,10 +203,12 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para ids=[ "Contacts stream with v2 field transformations", "Deals stream with v2 field transformations", - "DealsArchived stream with v2 field transformations" - ] + "DealsArchived stream with v2 field transformations", + ], ) -def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list): +def test_stream_read_with_legacy_field_transformation( + stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list +): stream = stream(**common_params) responses = [ { @@ -221,7 +223,7 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", "hs_v2_cumulative_time_in_prsopect": "1 month", "hs_v2_some_other_property_in_prospect": "Great property", - } + }, } | cursor_value ], @@ -246,44 +248,40 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v records = read_full_refresh(stream) assert records expected_record = { - "id": "test_id", - "created": "2022-02-25T16:43:11Z", - "properties": { - "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "hs_v2_latest_time_in_prospect": "1 month", - "hs_v2_cumulative_time_in_prsopect": "1 month", - "hs_v2_some_other_property_in_prospect": "Great property", - "hs_time_in_prospect": "1 month", - "hs_date_exited_prospect": "2024-02-01T00:00:00Z", - }, - "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "properties_hs_v2_latest_time_in_prospect": "1 month", - "properties_hs_v2_cumulative_time_in_prsopect": "1 month", - "properties_hs_v2_some_other_property_in_prospect": "Great property", - "properties_hs_time_in_prospect": "1 month", - "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", - } | cursor_value + "id": "test_id", + "created": "2022-02-25T16:43:11Z", + "properties": { + "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "hs_v2_latest_time_in_prospect": "1 month", + "hs_v2_cumulative_time_in_prsopect": "1 month", + "hs_v2_some_other_property_in_prospect": "Great property", + "hs_time_in_prospect": "1 month", + "hs_date_exited_prospect": "2024-02-01T00:00:00Z", + }, + "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "properties_hs_v2_latest_time_in_prospect": "1 month", + "properties_hs_v2_cumulative_time_in_prsopect": "1 month", + "properties_hs_v2_some_other_property_in_prospect": "Great property", + "properties_hs_time_in_prospect": "1 month", + "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", + } | cursor_value if isinstance(stream, Contacts): expected_record = expected_record | {"properties_hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} expected_record["properties"] = expected_record["properties"] | {"hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} else: - expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z" } - expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z" } + expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z"} + expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z"} assert records[0] == expected_record - @pytest.mark.parametrize("sync_mode", [SyncMode.full_refresh, SyncMode.incremental]) -def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): +def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): stream = DealSplits(**common_params) stream_state = { "type": "STREAM", - "stream": { - "stream_descriptor": { "name": "deal_splits" }, - "stream_state": { "updatedAt": "2021-01-01T00:00:00.000000Z" } - } + "stream": {"stream_descriptor": {"name": "deal_splits"}, "stream_state": {"updatedAt": "2021-01-01T00:00:00.000000Z"}}, } cursor_value = {"updatedAt": "2022-02-25T16:43:11Z"} responses = [ @@ -583,7 +581,7 @@ def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(req def test_get_custom_objects_metadata_success(requests_mock, custom_object_schema, expected_custom_object_json_schema, api): requests_mock.register_uri("GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) - for (entity, fully_qualified_name, schema, custom_properties) in api.get_custom_objects_metadata(): + for entity, fully_qualified_name, schema, custom_properties in api.get_custom_objects_metadata(): assert entity == "animals" assert fully_qualified_name == "p19936848_Animal" assert schema == expected_custom_object_json_schema diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py index 9bff520cf87e..942f0428b3f5 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py @@ -6,6 +6,7 @@ import json import pytest + from airbyte_cdk.models import ConfiguredAirbyteCatalog diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py index 6687d9e16f0c..1703f485ba35 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py @@ -7,9 +7,10 @@ import pendulum import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, ConfiguredAirbyteCatalog, Type from source_instagram.source import SourceInstagram +from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, ConfiguredAirbyteCatalog, Type + @pytest.fixture(name="state") def state_fixture() -> MutableMapping[str, Any]: @@ -35,12 +36,16 @@ def test_incremental_streams(self, configured_catalog, config, state): assert states, "insights should produce states" for state_msg in states: - stream_name, stream_state, state_keys_count = (state_msg.state.stream.stream_descriptor.name, - state_msg.state.stream.stream_state, - len(state_msg.state.stream.stream_state.dict())) + stream_name, stream_state, state_keys_count = ( + state_msg.state.stream.stream_descriptor.name, + state_msg.state.stream.stream_state, + len(state_msg.state.stream.stream_state.dict()), + ) assert stream_name == "user_insights", f"each state message should reference 'user_insights' stream, got {stream_name} instead" - assert isinstance(stream_state, AirbyteStateBlob), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" + assert isinstance( + stream_state, AirbyteStateBlob + ), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" assert state_keys_count == 2, f"Stream state should contain 2 partition keys, got {state_keys_count} instead" @staticmethod diff --git a/airbyte-integrations/connectors/source-instagram/main.py b/airbyte-integrations/connectors/source-instagram/main.py index 0a871930a015..7baf96d19171 100644 --- a/airbyte-integrations/connectors/source-instagram/main.py +++ b/airbyte-integrations/connectors/source-instagram/main.py @@ -4,5 +4,6 @@ from source_instagram.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index 16426efc1f9a..f9da2aa0df7d 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -8,15 +8,17 @@ import backoff import pendulum -from airbyte_cdk.entrypoint import logger from cached_property import cached_property from facebook_business import FacebookAdsApi from facebook_business.adobjects import user as fb_user from facebook_business.adobjects.iguser import IGUser from facebook_business.adobjects.page import Page from facebook_business.exceptions import FacebookRequestError + +from airbyte_cdk.entrypoint import logger from source_instagram.common import InstagramAPIException, retry_pattern + backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5, max_time=600) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py index 98efba83b47b..aad54c0e1311 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py @@ -11,6 +11,7 @@ from facebook_business.exceptions import FacebookRequestError from requests.status_codes import codes as status_codes + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py index 74ee90ff8cf8..98feb5e58125 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py @@ -5,6 +5,7 @@ from typing import Any, Dict, MutableMapping, Optional import requests + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -17,6 +18,7 @@ from .common import remove_params_from_url + GRAPH_URL = resolve_manifest(source=SourceInstagram()).record.data["manifest"]["definitions"]["base_requester"]["url_base"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index fe4a35309b95..33f9bcdb6d7b 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -4,11 +4,13 @@ from typing import Any, List, Mapping, Tuple import pendulum + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.core import Stream from source_instagram.api import InstagramAPI from source_instagram.streams import UserInsights + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index a01fa00b8056..397436a31bf0 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -9,10 +9,11 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum +from cached_property import cached_property + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from cached_property import cached_property from source_instagram.api import InstagramAPI from .common import remove_params_from_url diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index 92257eaab9e1..b290127f0de5 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -8,6 +8,7 @@ from pytest import fixture from source_instagram.api import InstagramAPI as API + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -57,12 +58,11 @@ def fb_account_response_fixture(account_id, some_config, requests_mock): "access_token": "access_token", "category": "Software company", "id": f"act_{account_id}", - "paging": {"cursors": { - "before": "cursor", - "after": "cursor"}}, + "paging": {"cursors": {"before": "cursor", "after": "cursor"}}, "summary": {"total_count": 1}, - "status_code": 200 - }] + "status_code": 200, + } + ] } } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py index b26f69f8a237..ef3ea86c51db 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py @@ -7,6 +7,7 @@ from typing import Any, List, MutableMapping + ACCESS_TOKEN = "test_access_token" ACCOUNT_ID = "111111111111111" diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py index 670d3c4c777e..f77b436893b4 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py @@ -6,6 +6,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "QVFIUlhOX3Rnbm5Y" diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py index 7ac44159c282..d10e4840f9bb 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py @@ -5,12 +5,14 @@ from typing import List, Optional, Union +from source_instagram.source import SourceInstagram + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.test.mock_http.request import HttpRequest -from source_instagram.source import SourceInstagram from .config import ACCOUNTS_FIELDS + GRAPH_URL = resolve_manifest(source=SourceInstagram()).record.data["manifest"]["definitions"]["base_requester"]["url_base"] @@ -94,4 +96,3 @@ def build(self) -> HttpRequest: def _item_path(self) -> str: path_for_resource = "/" if self._item_id_is_sub_path else "" return f"{self._item_id}{path_for_resource}" if self._item_id else "" - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py index 4a1746e422aa..67d14cad0a99 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py @@ -23,20 +23,7 @@ def build_response( def get_account_response() -> HttpResponse: response = { - "data": [ - { - "id": PAGE_ID, - "name": "AccountName", - "instagram_business_account": { - "id": BUSINESS_ACCOUNT_ID - } - } - ], - "paging": { - "cursors": { - "before": "before_token", - "after": "after_token" - } - } - } + "data": [{"id": PAGE_ID, "name": "AccountName", "instagram_business_account": {"id": BUSINESS_ACCOUNT_ID}}], + "paging": {"cursors": {"before": "before_token", "after": "after_token"}}, + } return build_response(body=response, status_code=HTTPStatus.OK) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py index 850356c62f49..218692c282f4 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py @@ -21,10 +21,8 @@ from .response_builder import get_account_response from .utils import config, read_output -_FIELDS = [ - "id", - "instagram_business_account" -] + +_FIELDS = ["id", "instagram_business_account"] _STREAM_NAME = "Api" @@ -46,7 +44,6 @@ def _record() -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -70,10 +67,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_accounts_with_no_instagram_business_account_field(self, http_mocker: HttpMocker) -> None: test = "not_instagram_business_account" mocked_response = json.dumps(find_template(f"api_for_{test}", __file__)) - http_mocker.get( - get_account_request().build(), - HttpResponse(mocked_response, 200) - ) + http_mocker.get(get_account_request().build(), HttpResponse(mocked_response, 200)) original_records = json.loads(mocked_response) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py index 429b75faa425..9319563fb463 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py @@ -23,13 +23,15 @@ from .response_builder import get_account_response from .utils import config, read_output + _FIELDS = [ "caption", "comments_count", "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -38,44 +40,21 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_CHILDREN_FIELDS = [ - "id", - "ig_id", - "media_type", - "media_url", - "owner", - "permalink", - "shortcode", - "thumbnail_url", - "timestamp", - "username" - ] - -_CHILDREN_IDS = [ - "07608776690540123", - "52896800415362123", - "39559889460059123", - "17359925580923123" - ] +_CHILDREN_FIELDS = ["id", "ig_id", "media_type", "media_url", "owner", "permalink", "shortcode", "thumbnail_url", "timestamp", "username"] + +_CHILDREN_IDS = ["07608776690540123", "52896800415362123", "39559889460059123", "17359925580923123"] _STREAM_NAME = "media" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(_FIELDS) -def _get_children_request(media_id:str) -> RequestBuilder: - return( - RequestBuilder.get_media_children_endpoint(item_id=media_id) - .with_fields(_CHILDREN_FIELDS) - ) +def _get_children_request(media_id: str) -> RequestBuilder: + return RequestBuilder.get_media_children_endpoint(item_id=media_id).with_fields(_CHILDREN_FIELDS) def _get_response() -> HttpResponseBuilder: @@ -93,8 +72,8 @@ def _record() -> RecordBuilder: record_id_path=FieldPath("id"), ) -class TestFullRefresh(TestCase): +class TestFullRefresh(TestCase): @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -162,15 +141,12 @@ def test_given_one_page_has_children_field(self, http_mocker: HttpMocker) -> Non get_account_request().build(), get_account_response(), ) - http_mocker.get( - _get_request().build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) - ) + http_mocker.get(_get_request().build(), HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200)) for children_id in _CHILDREN_IDS: http_mocker.get( _get_children_request(children_id).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py index 11902699b40b..bc95b4cbc934 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py @@ -6,6 +6,7 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( @@ -24,13 +25,15 @@ from .response_builder import get_account_response from .utils import config, read_output + PARENT_FIELDS = [ "caption", "comments_count", "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -39,9 +42,9 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_PARENT_STREAM_NAME = 'media' +_PARENT_STREAM_NAME = "media" _STREAM_NAME = "media_insights" @@ -54,28 +57,38 @@ MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS = "35076616084176125" MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "35076616084176126" -REELS = 'reels' -VIDEO_FEED = 'video_feed' -VIDEO = 'video' -CAROUSEL_ALBUM = 'carousel_album' -GENERAL_MEDIA = 'general_media' -ERROR_POSTED_BEFORE_BUSINESS = 'error_posted_before_business' -ERROR_WITH_WRONG_PERMISSIONS = 'error_with_wrong_permissions' -ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = 'error_with_wrong_permissions_code_10' +REELS = "reels" +VIDEO_FEED = "video_feed" +VIDEO = "video" +CAROUSEL_ALBUM = "carousel_album" +GENERAL_MEDIA = "general_media" +ERROR_POSTED_BEFORE_BUSINESS = "error_posted_before_business" +ERROR_WITH_WRONG_PERMISSIONS = "error_with_wrong_permissions" +ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "error_with_wrong_permissions_code_10" _MEDIA_IDS = { REELS: MEDIA_ID_REELS, VIDEO_FEED: MEDIA_ID_VIDEO_FEED, VIDEO: MEDIA_ID_VIDEO, CAROUSEL_ALBUM: MEDIA_ID_CAROUSEL_ALBUM, - GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA + GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA, } METRICS_GENERAL_MEDIA = ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"] _METRICS = { - MEDIA_ID_REELS: ["comments", "ig_reels_avg_watch_time", "ig_reels_video_view_total_time", "likes", "plays", "reach", "saved", "shares", - "ig_reels_aggregated_all_plays_count", "clips_replays_count"], + MEDIA_ID_REELS: [ + "comments", + "ig_reels_avg_watch_time", + "ig_reels_video_view_total_time", + "likes", + "plays", + "reach", + "saved", + "shares", + "ig_reels_aggregated_all_plays_count", + "clips_replays_count", + ], MEDIA_ID_VIDEO_FEED: ["impressions", "reach", "saved", "video_views"], MEDIA_ID_VIDEO: ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"], MEDIA_ID_CAROUSEL_ALBUM: ["impressions", "reach", "saved", "video_views", "shares", "follows", "profile_visits"], @@ -83,29 +96,22 @@ # Reusing general media metrics for error scenarios MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS: METRICS_GENERAL_MEDIA, MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS: METRICS_GENERAL_MEDIA, - MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA + MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA, } def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -114,15 +120,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -131,7 +135,6 @@ def _record(stream_name: str, test: str = None) -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -150,12 +153,14 @@ def test_instagram_insights_for_reels(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_REELS, metric=_METRICS[MEDIA_ID_REELS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -175,12 +180,14 @@ def test_instagram_insights_for_video_feed(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO_FEED, metric=_METRICS[MEDIA_ID_VIDEO_FEED]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -200,12 +207,14 @@ def test_instagram_insights_for_video(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO, metric=_METRICS[MEDIA_ID_VIDEO]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -225,12 +234,14 @@ def test_instagram_insights_carousel_album(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_CAROUSEL_ALBUM, metric=_METRICS[MEDIA_ID_CAROUSEL_ALBUM]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -250,12 +261,14 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -266,7 +279,6 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_insights_error_posted_before_business(self, http_mocker: HttpMocker) -> None: test = ERROR_POSTED_BEFORE_BUSINESS @@ -275,18 +287,19 @@ def test_instagram_insights_error_posted_before_business(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -305,18 +318,19 @@ def test_instagram_insights_error_with_wrong_permissions(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -336,18 +350,19 @@ def test_instagram_insights_error_with_wrong_permissions_code_10(self, http_mock get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py index abc137b78c71..ac8ec96a3e1c 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py @@ -22,6 +22,7 @@ from .response_builder import get_account_response from .utils import config, read_output + FIELDS = [ "caption", "id", @@ -35,18 +36,14 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] _STREAM_NAME = "stories" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(FIELDS) def _get_response() -> HttpResponseBuilder: @@ -66,7 +63,6 @@ def _record() -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py index 1e9e3f0f47aa..5e71182a2349 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py @@ -6,6 +6,7 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( @@ -24,6 +25,7 @@ from .response_builder import get_account_response from .utils import config, read_output + PARENT_FIELDS = [ "caption", "id", @@ -37,40 +39,32 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] -_PARENT_STREAM_NAME = 'stories' +_PARENT_STREAM_NAME = "stories" _STREAM_NAME = "story_insights" STORIES_ID = "3874523487643" STORIES_ID_ERROR_CODE_10 = "3874523487644" -HAPPY_PATH = 'story_insights_happy_path' -ERROR_10 = 'story_insights_error_code_10' +HAPPY_PATH = "story_insights_happy_path" +ERROR_10 = "story_insights_error_code_10" _METRICS = ["impressions", "reach", "replies", "follows", "profile_visits", "shares", "total_interactions"] - def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -79,15 +73,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -96,7 +88,6 @@ def _record(stream_name: str, test: str = None) -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -117,12 +108,14 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: # Mocking parent stream http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -133,7 +126,6 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: for metric in _METRICS: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocker) -> None: test = ERROR_10 @@ -143,18 +135,17 @@ def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocke ) # Mocking parent stream http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) # Good response http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200), ) # error 10 http_mocker.get( _get_child_request(media_id=STORIES_ID_ERROR_CODE_10, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py index a9a5d717fc0f..48dc09b91d6b 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py @@ -21,6 +21,7 @@ from .response_builder import get_account_response from .utils import config, read_output + _CURSOR_FIELD = "id" _STREAM_NAME = "user_lifetime_insights" @@ -29,28 +30,30 @@ def _get_request() -> RequestBuilder: return ( RequestBuilder.get_user_lifetime_insights_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_custom_param("metric", "follower_demographics").with_custom_param("period", "lifetime").with_custom_param("metric_type", "total_value").with_limit(100) + .with_custom_param("metric", "follower_demographics") + .with_custom_param("period", "lifetime") + .with_custom_param("metric_type", "total_value") + .with_limit(100) ) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), record_cursor_path=FieldPath(_CURSOR_FIELD), ) class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -76,4 +79,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) # each breakdown should produce a record assert len(output.records) == 3 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py index 2e50aae606f2..97c3519006fa 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py @@ -21,6 +21,7 @@ from .response_builder import get_account_response from .utils import config, read_output + _FIELDS = [ "id", "biography", @@ -31,36 +32,32 @@ "name", "profile_picture_url", "username", - "website" + "website", ] _STREAM_NAME = "users" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_fields(_FIELDS) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), ) class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -83,4 +80,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) assert len(output.records) == 1 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py index 5be5aef5c8bb..dd58fde5f31d 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py @@ -5,10 +5,11 @@ from typing import List, Optional +from source_instagram import SourceInstagram + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode -from source_instagram import SourceInstagram from .config import ConfigBuilder diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py index ecf06f7b64e5..4ccb16aade93 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py @@ -1,77 +1,53 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. children_record = { - "children": { - "data": [ - { - "id": "7608776690540" - }, - { - "id": "2896800415362" - }, - { - "id": "9559889460059" - }, - { - "id": "7359925580923" - } - ] - } + "children": {"data": [{"id": "7608776690540"}, {"id": "2896800415362"}, {"id": "9559889460059"}, {"id": "7359925580923"}]} } expected_children_transformed = { - "children": - [ - { + "children": [ + { "id": "7608776690540", "ig_id": "2521545917836833225", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=tTUSyCXbN40Q7kNvgF-k3H6&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYAe4PKmuen4Dryt4sMEbfvrW2_eANbY1AEdl7gHG0a3mw&oe=66701C8F", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "2896800415362", "ig_id": "2521545917736276706", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=100&ccb=1-7&_nc_sid=18de74&_nc_ohc=9qJ5-fOc9lcQ7kNvgFirW6U&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBSuGRqMEzjxHri30L5NDs0irt7_7h-arKTYS8inrL56g&oe=66702E9A", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "9559889460059", "ig_id": "2521545917845406325", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=110&ccb=1-7&_nc_sid=18de74&_nc_ohc=QOtZzdxFjusQ7kNvgEqsuc2&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBPBGVS3NYW-h8oLwam_rWub-mE-9MLGc1EDVHtLJ2DBQ&oe=66702DE1", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "7359925580923", "ig_id": "2521545555591565193", "media_type": "VIDEO", "media_url": "https://fake_url?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNhcm91c2VsX2l0ZW0udW5rbm93bi1DMy40ODAuZGFzaF9iYXNlbGluZV8xX3YxIn0&_nc_ht=fakecontent&_nc_cat=108&vs=863753484982045_2117350142", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "thumbnail_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=pJkRskDC80UQ7kNvgFn3i4H&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBKK27CU9dvjiqPi9a4JKUIICp26HZ074-vgz0OVKFkbw&oe=66702104", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - } - ] + "username": "username", + }, + ] } clear_url_record = { @@ -85,293 +61,66 @@ } breakdowns_record = { - "name": "follower_demographics", - "period": "lifetime", - "title": "Follower demographics", - "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", - "total_value": { - "breakdowns": [ + "name": "follower_demographics", + "period": "lifetime", + "title": "Follower demographics", + "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", + "total_value": { + "breakdowns": [ { - "dimension_keys": [ - "city" - ], - "results": [ - { - "dimension_values": [ - "London, England" - ], - "value": 263 - }, - { - "dimension_values": [ - "Sydney, New South Wales" - ], - "value": 467 - }, - { - "dimension_values": [ - "Algiers, Algiers Province" - ], - "value": 58 - }, - { - "dimension_values": [ - "Casablanca, Grand Casablanca" - ], - "value": 71 - }, - { - "dimension_values": [ - "São Paulo, São Paulo (state)" - ], - "value": 139 - }, - { - "dimension_values": [ - "Rio de Janeiro, Rio de Janeiro (state)" - ], - "value": 44 - }, - { - "dimension_values": [ - "Perth, Western Australia" - ], - "value": 180 - }, - { - "dimension_values": [ - "Berlin, Berlin" - ], - "value": 47 - }, - { - "dimension_values": [ - "Kolkata, West Bengal" - ], - "value": 85 - }, - { - "dimension_values": [ - "Phoenix, Arizona" - ], - "value": 39 - }, - { - "dimension_values": [ - "Lagos, Lagos State" - ], - "value": 40 - }, - { - "dimension_values": [ - "Dublin, Dublin" - ], - "value": 65 - }, - { - "dimension_values": [ - "Pune, Maharashtra" - ], - "value": 72 - }, - { - "dimension_values": [ - "Wollongong, New South Wales" - ], - "value": 43 - }, - { - "dimension_values": [ - "Christchurch, Canterbury" - ], - "value": 42 - }, - { - "dimension_values": [ - "Jakarta, Jakarta" - ], - "value": 46 - }, - { - "dimension_values": [ - "Pretoria, Gauteng" - ], - "value": 54 - }, - { - "dimension_values": [ - "Buenos Aires, Ciudad Autónoma de Buenos Aires" - ], - "value": 41 - }, - { - "dimension_values": [ - "Gold Coast, Queensland" - ], - "value": 98 - }, - { - "dimension_values": [ - "Sunshine Coast, Queensland" - ], - "value": 37 - }, - { - "dimension_values": [ - "Melbourne, Victoria" - ], - "value": 338 - }, - { - "dimension_values": [ - "Gurugram, Haryana" - ], - "value": 52 - }, - { - "dimension_values": [ - "Delhi, Delhi" - ], - "value": 194 - }, - { - "dimension_values": [ - "Los Angeles, California" - ], - "value": 66 - }, - { - "dimension_values": [ - "Madrid, Comunidad de Madrid" - ], - "value": 65 - }, - { - "dimension_values": [ - "Lahore, Punjab" - ], - "value": 41 - }, - { - "dimension_values": [ - "Brisbane, Queensland" - ], - "value": 160 - }, - { - "dimension_values": [ - "Adelaide, South Australia" - ], - "value": 93 - }, - { - "dimension_values": [ - "Canberra, Australian Capital Territory" - ], - "value": 45 - }, - { - "dimension_values": [ - "Lima, Lima Region" - ], - "value": 43 - }, - { - "dimension_values": [ - "Istanbul, Istanbul Province" - ], - "value": 57 - }, - { - "dimension_values": [ - "Toronto, Ontario" - ], - "value": 40 - }, - { - "dimension_values": [ - "Chennai, Tamil Nadu" - ], - "value": 82 - }, - { - "dimension_values": [ - "Mexico City, Distrito Federal" - ], - "value": 66 - }, - { - "dimension_values": [ - "Auckland, Auckland Region" - ], - "value": 98 - }, - { - "dimension_values": [ - "Cape Town, Western Cape" - ], - "value": 172 - }, - { - "dimension_values": [ - "New York, New York" - ], - "value": 139 - }, - { - "dimension_values": [ - "Cairo, Cairo Governorate" - ], - "value": 45 - }, - { - "dimension_values": [ - "Dubai, Dubai" - ], - "value": 57 - }, - { - "dimension_values": [ - "Santiago, Santiago Metropolitan Region" - ], - "value": 73 - }, - { - "dimension_values": [ - "Mumbai, Maharashtra" - ], - "value": 195 - }, - { - "dimension_values": [ - "Bangalore, Karnataka" - ], - "value": 195 - }, - { - "dimension_values": [ - "Nairobi, Nairobi" - ], - "value": 50 - }, - { - "dimension_values": [ - "Johannesburg, Gauteng" - ], - "value": 50 - }, - { - "dimension_values": [ - "Hyderabad, Telangana" - ], - "value": 49 - } - ] + "dimension_keys": ["city"], + "results": [ + {"dimension_values": ["London, England"], "value": 263}, + {"dimension_values": ["Sydney, New South Wales"], "value": 467}, + {"dimension_values": ["Algiers, Algiers Province"], "value": 58}, + {"dimension_values": ["Casablanca, Grand Casablanca"], "value": 71}, + {"dimension_values": ["São Paulo, São Paulo (state)"], "value": 139}, + {"dimension_values": ["Rio de Janeiro, Rio de Janeiro (state)"], "value": 44}, + {"dimension_values": ["Perth, Western Australia"], "value": 180}, + {"dimension_values": ["Berlin, Berlin"], "value": 47}, + {"dimension_values": ["Kolkata, West Bengal"], "value": 85}, + {"dimension_values": ["Phoenix, Arizona"], "value": 39}, + {"dimension_values": ["Lagos, Lagos State"], "value": 40}, + {"dimension_values": ["Dublin, Dublin"], "value": 65}, + {"dimension_values": ["Pune, Maharashtra"], "value": 72}, + {"dimension_values": ["Wollongong, New South Wales"], "value": 43}, + {"dimension_values": ["Christchurch, Canterbury"], "value": 42}, + {"dimension_values": ["Jakarta, Jakarta"], "value": 46}, + {"dimension_values": ["Pretoria, Gauteng"], "value": 54}, + {"dimension_values": ["Buenos Aires, Ciudad Autónoma de Buenos Aires"], "value": 41}, + {"dimension_values": ["Gold Coast, Queensland"], "value": 98}, + {"dimension_values": ["Sunshine Coast, Queensland"], "value": 37}, + {"dimension_values": ["Melbourne, Victoria"], "value": 338}, + {"dimension_values": ["Gurugram, Haryana"], "value": 52}, + {"dimension_values": ["Delhi, Delhi"], "value": 194}, + {"dimension_values": ["Los Angeles, California"], "value": 66}, + {"dimension_values": ["Madrid, Comunidad de Madrid"], "value": 65}, + {"dimension_values": ["Lahore, Punjab"], "value": 41}, + {"dimension_values": ["Brisbane, Queensland"], "value": 160}, + {"dimension_values": ["Adelaide, South Australia"], "value": 93}, + {"dimension_values": ["Canberra, Australian Capital Territory"], "value": 45}, + {"dimension_values": ["Lima, Lima Region"], "value": 43}, + {"dimension_values": ["Istanbul, Istanbul Province"], "value": 57}, + {"dimension_values": ["Toronto, Ontario"], "value": 40}, + {"dimension_values": ["Chennai, Tamil Nadu"], "value": 82}, + {"dimension_values": ["Mexico City, Distrito Federal"], "value": 66}, + {"dimension_values": ["Auckland, Auckland Region"], "value": 98}, + {"dimension_values": ["Cape Town, Western Cape"], "value": 172}, + {"dimension_values": ["New York, New York"], "value": 139}, + {"dimension_values": ["Cairo, Cairo Governorate"], "value": 45}, + {"dimension_values": ["Dubai, Dubai"], "value": 57}, + {"dimension_values": ["Santiago, Santiago Metropolitan Region"], "value": 73}, + {"dimension_values": ["Mumbai, Maharashtra"], "value": 195}, + {"dimension_values": ["Bangalore, Karnataka"], "value": 195}, + {"dimension_values": ["Nairobi, Nairobi"], "value": 50}, + {"dimension_values": ["Johannesburg, Gauteng"], "value": 50}, + {"dimension_values": ["Hyderabad, Telangana"], "value": 49}, + ], } - ] - }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + ] + }, + "id": "17841457631192237/insights/follower_demographics/lifetime", +} expected_breakdown_record_transformed = { "name": "follower_demographics", @@ -379,153 +128,121 @@ "title": "Follower demographics", "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", "value": { - "London, England": 263, - "Sydney, New South Wales": 467, - "Algiers, Algiers Province": 58, - "Casablanca, Grand Casablanca": 71, - "São Paulo, São Paulo (state)": 139, - "Rio de Janeiro, Rio de Janeiro (state)": 44, - "Perth, Western Australia": 180, - "Berlin, Berlin": 47, - "Kolkata, West Bengal": 85, - "Phoenix, Arizona": 39, - "Lagos, Lagos State": 40, - "Dublin, Dublin": 65, - "Pune, Maharashtra": 72, - "Wollongong, New South Wales": 43, - "Christchurch, Canterbury": 42, - "Jakarta, Jakarta": 46, - "Pretoria, Gauteng": 54, - "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, - "Gold Coast, Queensland": 98, - "Sunshine Coast, Queensland": 37, - "Melbourne, Victoria": 338, - "Gurugram, Haryana": 52, - "Delhi, Delhi": 194, - "Los Angeles, California": 66, - "Madrid, Comunidad de Madrid": 65, - "Lahore, Punjab": 41, - "Brisbane, Queensland": 160, - "Adelaide, South Australia": 93, - "Canberra, Australian Capital Territory": 45, - "Lima, Lima Region": 43, - "Istanbul, Istanbul Province": 57, - "Toronto, Ontario": 40, - "Chennai, Tamil Nadu": 82, - "Mexico City, Distrito Federal": 66, - "Auckland, Auckland Region": 98, - "Cape Town, Western Cape": 172, - "New York, New York": 139, - "Cairo, Cairo Governorate": 45, - "Dubai, Dubai": 57, - "Santiago, Santiago Metropolitan Region": 73, - "Mumbai, Maharashtra": 195, - "Bangalore, Karnataka": 195, - "Nairobi, Nairobi": 50, - "Johannesburg, Gauteng": 50, - "Hyderabad, Telangana": 49 + "London, England": 263, + "Sydney, New South Wales": 467, + "Algiers, Algiers Province": 58, + "Casablanca, Grand Casablanca": 71, + "São Paulo, São Paulo (state)": 139, + "Rio de Janeiro, Rio de Janeiro (state)": 44, + "Perth, Western Australia": 180, + "Berlin, Berlin": 47, + "Kolkata, West Bengal": 85, + "Phoenix, Arizona": 39, + "Lagos, Lagos State": 40, + "Dublin, Dublin": 65, + "Pune, Maharashtra": 72, + "Wollongong, New South Wales": 43, + "Christchurch, Canterbury": 42, + "Jakarta, Jakarta": 46, + "Pretoria, Gauteng": 54, + "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, + "Gold Coast, Queensland": 98, + "Sunshine Coast, Queensland": 37, + "Melbourne, Victoria": 338, + "Gurugram, Haryana": 52, + "Delhi, Delhi": 194, + "Los Angeles, California": 66, + "Madrid, Comunidad de Madrid": 65, + "Lahore, Punjab": 41, + "Brisbane, Queensland": 160, + "Adelaide, South Australia": 93, + "Canberra, Australian Capital Territory": 45, + "Lima, Lima Region": 43, + "Istanbul, Istanbul Province": 57, + "Toronto, Ontario": 40, + "Chennai, Tamil Nadu": 82, + "Mexico City, Distrito Federal": 66, + "Auckland, Auckland Region": 98, + "Cape Town, Western Cape": 172, + "New York, New York": 139, + "Cairo, Cairo Governorate": 45, + "Dubai, Dubai": 57, + "Santiago, Santiago Metropolitan Region": 73, + "Mumbai, Maharashtra": 195, + "Bangalore, Karnataka": 195, + "Nairobi, Nairobi": 50, + "Johannesburg, Gauteng": 50, + "Hyderabad, Telangana": 49, }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + "id": "17841457631192237/insights/follower_demographics/lifetime", +} insights_record = { "data": [ - { - "name": "comments", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title1", - "description": "Description1.", - "id": "insta_id/insights/comments/lifetime" - }, - { - "name": "ig_reels_avg_watch_time", - "period": "lifetime", - "values": [ - { - "value": 11900 - } - ], - "title": "2", - "description": "Description2.", - "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime" - }, - { - "name": "ig_reels_video_view_total_time", - "period": "lifetime", - "values": [ - { - "value": 25979677 - } - ], - "title": "title3", - "description": "Description3.", - "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime" - }, - { - "name": "likes", - "period": "lifetime", - "values": [ - { - "value": 102 - } - ], - "title": "title4", - "description": "Description4.", - "id": "insta_id/insights/likes/lifetime" - }, - { - "name": "plays", - "period": "lifetime", - "values": [ - { - "value": 2176 - } - ], - "title": "title5", - "description": "Description5.", - "id": "insta_id/insights/plays/lifetime" - }, - { - "name": "reach", - "period": "lifetime", - "values": [ - { - "value": 1842 - } - ], - "title": "title6", - "description": "Description6.", - "id": "insta_id/insights/reach/lifetime" - }, - { - "name": "saved", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title7", - "description": "Description7.", - "id": "insta_id/insights/saved/lifetime" - }, - { - "name": "shares", - "period": "lifetime", - "values": [ - { - "value": 1 - } - ], - "title": "title8", - "description": "Description8.", - "id": "insta_id/insights/shares/lifetime" - } + { + "name": "comments", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title1", + "description": "Description1.", + "id": "insta_id/insights/comments/lifetime", + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [{"value": 11900}], + "title": "2", + "description": "Description2.", + "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime", + }, + { + "name": "ig_reels_video_view_total_time", + "period": "lifetime", + "values": [{"value": 25979677}], + "title": "title3", + "description": "Description3.", + "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime", + }, + { + "name": "likes", + "period": "lifetime", + "values": [{"value": 102}], + "title": "title4", + "description": "Description4.", + "id": "insta_id/insights/likes/lifetime", + }, + { + "name": "plays", + "period": "lifetime", + "values": [{"value": 2176}], + "title": "title5", + "description": "Description5.", + "id": "insta_id/insights/plays/lifetime", + }, + { + "name": "reach", + "period": "lifetime", + "values": [{"value": 1842}], + "title": "title6", + "description": "Description6.", + "id": "insta_id/insights/reach/lifetime", + }, + { + "name": "saved", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title7", + "description": "Description7.", + "id": "insta_id/insights/saved/lifetime", + }, + { + "name": "shares", + "period": "lifetime", + "values": [{"value": 1}], + "title": "title8", + "description": "Description8.", + "id": "insta_id/insights/shares/lifetime", + }, ] } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 4f371e0fc9cf..e1bcacfc5dc7 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -5,6 +5,8 @@ import logging +from source_instagram.source import SourceInstagram + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.models import ( AirbyteStream, @@ -14,7 +16,7 @@ DestinationSyncMode, SyncMode, ) -from source_instagram.source import SourceInstagram + logger = logging.getLogger("airbyte") @@ -24,23 +26,11 @@ account_url_response = { - "data": [ - { - "id": "page_id", - "name": "Airbyte", - "instagram_business_account": { - "id": "instagram_business_account_id" - } - } - ], - "paging": { - "cursors": { - "before": "before", - "after": "after" - } - } + "data": [{"id": "page_id", "name": "Airbyte", "instagram_business_account": {"id": "instagram_business_account_id"}}], + "paging": {"cursors": {"before": "before", "after": "after"}}, } + def test_check_connection_ok(api, requests_mock, some_config): requests_mock.register_uri("GET", account_url, [{"json": account_url_response}]) ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 58fe97e5002f..74fc74357524 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -6,11 +6,13 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from source_instagram.streams import DatetimeTransformerMixin, InstagramStream, UserInsights from utils import read_full_refresh, read_incremental +from airbyte_cdk.models import SyncMode + + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -30,7 +32,6 @@ def test_state_is_not_outdated(api, config): assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) - def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" @@ -42,7 +43,6 @@ def test_user_insights_read(api, config, user_insight_data, requests_mock): assert records - @pytest.mark.parametrize( "values,slice_dates,expected", [ diff --git a/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instatus/main.py b/airbyte-integrations/connectors/source-instatus/main.py index 0e0c5af556b5..e2352fc98eb1 100644 --- a/airbyte-integrations/connectors/source-instatus/main.py +++ b/airbyte-integrations/connectors/source-instatus/main.py @@ -4,5 +4,6 @@ from source_instatus.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/components.py b/airbyte-integrations/connectors/source-instatus/source_instatus/components.py index 62c6561c19cb..e589d1f07bb4 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/components.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, List, Mapping, Optional import dpath.util + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig, SubstreamPartitionRouter from airbyte_cdk.sources.declarative.transformations import AddFields diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py index ade50f0a9cdd..ccf55560c448 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_instatus import SourceInstatus +from airbyte_cdk.entrypoint import launch + def run(): source = SourceInstatus() diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/source.py b/airbyte-integrations/connectors/source-instatus/source_instatus/source.py index dda77f207308..2f076f8e1f26 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/source.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py index f09c7ff5e83e..7f6d4b9ceb85 100644 --- a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py @@ -5,10 +5,11 @@ from unittest.mock import MagicMock import pytest +from source_instatus.components import ListAddFields, UpdatesSubstreamPartitionRouter + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition -from source_instatus.components import ListAddFields, UpdatesSubstreamPartitionRouter @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-intercom/components.py b/airbyte-integrations/connectors/source-intercom/components.py index 4d3670c9490d..76de447344c9 100644 --- a/airbyte-integrations/connectors/source-intercom/components.py +++ b/airbyte-integrations/connectors/source-intercom/components.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, Optional, Union import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -18,6 +19,7 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution + RequestInput = Union[str, Mapping[str, str]] @@ -199,7 +201,6 @@ def read_parent_stream( cursor_field: Optional[str], stream_state: Mapping[str, Any], ) -> Iterable[Mapping[str, Any]]: - self.parent_stream.state = stream_state parent_stream_slices_gen = self.parent_stream.stream_slices( diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py index a2a8db7650e5..92dcc44da94e 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py @@ -6,6 +6,7 @@ import pytest import requests + from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from airbyte_cdk.sources.streams import Stream @@ -75,8 +76,8 @@ def test_sub_slicer(components_module, last_record, expected, records): ], ) def test_rate_limiter(components_module, rate_limit_header, backoff_time): - IntercomRateLimiter = components_module.IntercomRateLimiter + def check_backoff_time(t): """A replacer for original `IntercomRateLimiter.backoff_time`""" assert backoff_time == t, f"Expected {backoff_time}, got {t}" diff --git a/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py index 51f9b12de27d..e023a7a92b30 100644 --- a/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-iterable/main.py b/airbyte-integrations/connectors/source-iterable/main.py index eef7d894cbc4..5b07bc8070e3 100644 --- a/airbyte-integrations/connectors/source-iterable/main.py +++ b/airbyte-integrations/connectors/source-iterable/main.py @@ -4,5 +4,6 @@ from source_iterable.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/components.py b/airbyte-integrations/connectors/source-iterable/source_iterable/components.py index 19d218fff055..71f216c5af37 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/components.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 83fa25aa174b..f1e78543b423 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -47,6 +47,7 @@ WebPushSendSkip, ) + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. @@ -54,6 +55,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourceIterable(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index dae02ef017f3..b7e232e5ab56 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -10,18 +10,20 @@ import pendulum import requests +from pendulum.datetime import DateTime +from requests import HTTPError +from requests.exceptions import ChunkedEncodingError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import CheckpointMixin, package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from pendulum.datetime import DateTime -from requests import HTTPError -from requests.exceptions import ChunkedEncodingError from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice from source_iterable.utils import dateutil_parse + EVENT_ROWS_LIMIT = 200 CAMPAIGNS_PER_REQUEST = 20 @@ -202,7 +204,6 @@ def request_params( stream_slice: StreamSlice, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state) params.update( { @@ -250,7 +251,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) return [StreamSlice(start_datetime, self._end_date or pendulum.now("UTC"))] @@ -268,7 +268,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) return RangeSliceGenerator(start_datetime, self._end_date) @@ -303,7 +302,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) self._adjustable_generator = AdjustableSliceGenerator(start_datetime, self._end_date) return self._adjustable_generator @@ -318,7 +316,6 @@ def read_records( start_time = pendulum.now() for _ in range(self.CHUNKED_ENCODING_ERROR_RETRIES): try: - self.logger.info( f"Processing slice of {(stream_slice.end_date - stream_slice.start_date).total_days()} days for stream {self.name}" ) diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py index 0210920dea23..f9053d5b44ad 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py @@ -7,6 +7,7 @@ import pytest import responses + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py index 90bd5b6ab286..3e54136178d5 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py @@ -12,11 +12,13 @@ import pendulum import pytest import responses -from airbyte_cdk.models import Type as MessageType from requests.exceptions import ChunkedEncodingError from source_iterable.slice_generators import AdjustableSliceGenerator from source_iterable.source import SourceIterable +from airbyte_cdk.models import Type as MessageType + + TEST_START_DATE = "2020" diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py index 0353d32c22b3..15bdab16414d 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py @@ -8,9 +8,10 @@ import pendulum import pytest import responses +from source_iterable.source import SourceIterable + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_iterable.source import SourceIterable @pytest.fixture diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py index 661e898d15de..09a49584e23a 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py @@ -4,9 +4,10 @@ import io import requests -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonlDecoder from source_iterable.components import EventsRecordExtractor +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonlDecoder + def test_events_extraction(): mock_response = requests.Response() diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py index 5c4bcf85a4e9..ff5890eb4030 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py @@ -9,6 +9,7 @@ import pytest from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator + TEST_DATE = pendulum.parse("2020-01-01") diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py index e7ed3e0f75ca..0c44168c77cb 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py @@ -7,9 +7,10 @@ import pytest import requests import responses +from source_iterable.source import SourceIterable + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_iterable.source import SourceIterable @responses.activate diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index 1c4d2347ba3d..4342e91af6b9 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -6,13 +6,14 @@ import pytest import requests import responses -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice from source_iterable.source import SourceIterable from source_iterable.streams import Campaigns, CampaignsMetrics, Templates from source_iterable.utils import dateutil_parse +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice + def test_campaigns_metrics_csv(): csv_string = "a,b,c,d\n1, 2,,3\n6,,1, 2\n" @@ -190,7 +191,6 @@ def test_path(config, stream, date, slice, expected_path): def test_campaigns_metrics_parse_response(): - stream = CampaignsMetrics(authenticator=None, start_date="2019-10-10T00:00:00") with responses.RequestsMock() as rsps: rsps.add( diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/main.py b/airbyte-integrations/connectors/source-jina-ai-reader/main.py index 33085d47ebf9..58af77ac1dd2 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/main.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/main.py @@ -5,5 +5,6 @@ from source_jina_ai_reader.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py index 7604776518b5..ba509deaa9ed 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py index bf73db8f9ed6..392ff158b075 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py index 2edcb5eb27c7..9825027db046 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py @@ -5,6 +5,7 @@ from source_jina_ai_reader.config_migration import JinaAiReaderConfigMigration from source_jina_ai_reader.source import SourceJinaAiReader + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" @@ -16,7 +17,7 @@ def test_should_migrate(): def test__modify_and_save(): source = SourceJinaAiReader() user_config = {"search_prompt": "What is AI"} - expected = {"search_prompt": "What%20is%20AI" } + expected = {"search_prompt": "What%20is%20AI"} modified_config = JinaAiReaderConfigMigration.modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config=user_config) assert modified_config["search_prompt"] == expected["search_prompt"] assert modified_config.get("search_prompt") diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py index e4e2e8337782..fb8a0c348a1d 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py @@ -7,7 +7,6 @@ from base64 import b64encode from typing import Any, List, Mapping -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from streams import ( DashboardsGenerator, FiltersGenerator, @@ -29,6 +28,8 @@ WorkflowsGenerator, ) +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + class Generator: base_config_path = "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py index 5a295903035b..365af929548f 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py @@ -8,7 +8,6 @@ from typing import Any, Mapping, Optional import requests -from airbyte_cdk.models import SyncMode from source_jira.streams import ( Dashboards, Filters, @@ -31,6 +30,8 @@ WorkflowSchemes, ) +from airbyte_cdk.models import SyncMode + class GeneratorMixin: def get_generate_headers(self): diff --git a/airbyte-integrations/connectors/source-jira/main.py b/airbyte-integrations/connectors/source-jira/main.py index 1885b3974def..528d513d7488 100644 --- a/airbyte-integrations/connectors/source-jira/main.py +++ b/airbyte-integrations/connectors/source-jira/main.py @@ -4,5 +4,6 @@ from source_jira.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py b/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py index 2a30cc666541..ff56146ada56 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py @@ -3,9 +3,10 @@ from dataclasses import dataclass from typing import Any, List, Mapping -from airbyte_cdk.sources.declarative.extractors import DpathExtractor from requests_cache import Response +from airbyte_cdk.sources.declarative.extractors import DpathExtractor + @dataclass class LabelsRecordExtractor(DpathExtractor): diff --git a/airbyte-integrations/connectors/source-jira/source_jira/run.py b/airbyte-integrations/connectors/source-jira/source_jira/run.py index 537430466f7a..8f5465ccd6c7 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/run.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/run.py @@ -7,10 +7,11 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_jira import SourceJira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index c72c2d5664fa..1e39ab123080 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -5,6 +5,9 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from pydantic import ValidationError +from requests.exceptions import InvalidURL + from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource @@ -12,8 +15,6 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from pydantic import ValidationError -from requests.exceptions import InvalidURL from .streams import IssueFields, Issues, PullRequests from .utils import read_full_refresh diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 1d9f622bd4a7..b9ef14e0cce3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -12,6 +12,8 @@ import pendulum import requests +from requests.exceptions import HTTPError + from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import CheckpointMixin, Stream from airbyte_cdk.sources.streams.checkpoint.checkpoint_reader import FULL_REFRESH_COMPLETE_STATE @@ -21,11 +23,11 @@ from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from requests.exceptions import HTTPError from source_jira.type_transfromer import DateTimeTransformer from .utils import read_full_refresh, read_incremental, safe_max + API_VERSION = 3 @@ -173,7 +175,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class FullRefreshJiraStream(JiraStream): - """ This is a temporary solution to avoid incorrect state handling. See comments below for more info: diff --git a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py index 16fdfb000a4c..efb3bca61278 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.utils.transform import TypeTransformer + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-jira/source_jira/utils.py b/airbyte-integrations/connectors/source-jira/source_jira/utils.py index f85c7363ff97..af6adbcf9108 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/utils.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/utils.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.streams import Stream + _NO_STATE: MutableMapping[str, Any] = {} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py index 215560d88cf1..b25b3acd3ba2 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py @@ -11,6 +11,7 @@ from responses import matchers from source_jira.source import SourceJira + ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py index c0306ea87a4a..2d8f2b212741 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py @@ -6,6 +6,8 @@ from unittest import TestCase import freezegun +from source_jira import SourceJira + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -20,7 +22,7 @@ ) from airbyte_cdk.test.state_builder import StateBuilder from integration.config import ConfigBuilder -from source_jira import SourceJira + _STREAM_NAME = "issues" _API_TOKEN = "api_token" @@ -40,6 +42,7 @@ def _response_template() -> Dict[str, Any]: with open(os.path.join(os.path.dirname(__file__), "..", "responses", "issues.json")) as response_file_handler: return json.load(response_file_handler) + def _create_response() -> HttpResponseBuilder: return create_response_builder( response_template=_response_template(), @@ -60,15 +63,19 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: config = _create_config().build() datetime_with_timezone = "2023-11-01T00:00:00.000-0800" timestamp_with_timezone = 1698825600000 - state = StateBuilder().with_stream_state( - "issues", - { - "use_global_cursor":False, - "state": {"updated": datetime_with_timezone}, - "lookback_window": 2, - "states": [{"partition":{"parent_slice":{},"project_id":"10025"},"cursor":{"updated": datetime_with_timezone}}] - } - ).build() + state = ( + StateBuilder() + .with_stream_state( + "issues", + { + "use_global_cursor": False, + "state": {"updated": datetime_with_timezone}, + "lookback_window": 2, + "states": [{"partition": {"parent_slice": {}, "project_id": "10025"}, "cursor": {"updated": datetime_with_timezone}}], + }, + ) + .build() + ) http_mocker.get( HttpRequest( f"https://{_DOMAIN}/rest/api/3/search", @@ -77,7 +84,7 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: "jql": f"updated >= {timestamp_with_timezone} ORDER BY updated asc", "expand": "renderedFields,transitions,changelog", "maxResults": "50", - } + }, ), _create_response().with_record(_create_record()).with_record(_create_record()).build(), ) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py index e24edba48d2f..50e20916d093 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py @@ -4,10 +4,11 @@ import pytest import requests -from airbyte_cdk.sources.declarative.types import StreamSlice from source_jira.components.extractors import LabelsRecordExtractor from source_jira.components.partition_routers import SprintIssuesSubstreamPartitionRouter +from airbyte_cdk.sources.declarative.types import StreamSlice + @pytest.mark.parametrize( "json_response, expected_output", diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py index be35d1a7f0d5..e072cc6616ed 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py @@ -6,9 +6,10 @@ import pytest import responses -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_jira.source import SourceJira +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @responses.activate def test_streams(config): diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index b6d05c6f4fbc..fd4f10dfd98a 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -5,16 +5,17 @@ import pendulum import pytest import responses -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import read -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from conftest import find_stream from responses import matchers from source_jira.source import SourceJira from source_jira.streams import IssueFields, Issues, PullRequests from source_jira.utils import read_full_refresh, read_incremental +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @responses.activate def test_application_roles_stream_401_error(config, caplog): @@ -50,9 +51,7 @@ def test_application_roles_stream_http_error(config, application_roles_response) responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/applicationrole", json={"error": "not found"}, status=404) stream = find_stream("application_roles", config) - with pytest.raises( - AirbyteTracedException, match="Not found. The requested resource was not found on the server" - ): + with pytest.raises(AirbyteTracedException, match="Not found. The requested resource was not found on the server"): list(read_full_refresh(stream)) @@ -82,10 +81,7 @@ def test_board_stream_forbidden(config, boards_response, caplog): ) stream = find_stream("boards", config) - with pytest.raises( - AirbyteTracedException, - match="Forbidden. You don't have permission to access this resource." - ): + with pytest.raises(AirbyteTracedException, match="Forbidden. You don't have permission to access this resource."): list(read_full_refresh(stream)) @@ -96,7 +92,7 @@ def test_dashboards_stream(config, dashboards_response): f"https://{config['domain']}/rest/api/3/dashboard", json=dashboards_response, ) - + stream = find_stream("dashboards", config) records = list(read_full_refresh(stream)) @@ -132,7 +128,7 @@ def test_groups_stream(config, groups_response): def test_issues_fields_stream(config, mock_fields_response): stream = find_stream("issue_fields", config) records = list(read_full_refresh(stream)) - + assert len(records) == 6 assert len(responses.calls) == 1 @@ -149,7 +145,7 @@ def test_python_issues_fields_ids_by_name(config, mock_fields_response): "Issue Type": ["issuetype"], "Parent": ["parent"], "Issue Type2": ["issuetype2"], - "Issue Type3": ["issuetype3"] + "Issue Type3": ["issuetype3"], } assert expected_ids_by_name == stream.field_ids_by_name() @@ -400,7 +396,9 @@ def test_screen_tabs_stream(config, mock_screen_response, screen_tabs_response): @responses.activate def test_sprints_stream(config, mock_board_response, mock_sprints_response): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build() + ) assert len(output.records) == 3 assert len(responses.calls) == 4 @@ -417,7 +415,7 @@ def test_board_does_not_support_sprints(config, mock_board_response, sprints_res responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50", json={"errorMessages": ["The board does not support sprints"], "errors": {}}, - status=400 + status=400, ) responses.add( responses.GET, @@ -443,7 +441,11 @@ def test_sprint_issues_stream(config, mock_board_response, mock_fields_response, json=sprints_issues_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build(), + ) assert len(output.records) == 3 assert len(responses.calls) == 8 @@ -585,7 +587,7 @@ def test_avatars_stream_should_retry(config, caplog): responses.GET, f"https://{config['domain']}/rest/api/3/avatar/{slice}/system", json={"errorMessages": ["The error message"], "errors": {}}, - status=400 + status=400, ) stream = find_stream("avatars", config) @@ -622,9 +624,11 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project assert "non_empty_field" in records[0]["fields"] assert len(responses.calls) == 3 - error_message = ("Stream `issues`. An error occurred, details: The user doesn't have " - 'permission to the project. Please grant the user to the project. Errors: ' - '["The value \'3\' does not exist for the field \'project\'."]') + error_message = ( + "Stream `issues`. An error occurred, details: The user doesn't have " + "permission to the project. Please grant the user to the project. Errors: " + "[\"The value '3' does not exist for the field 'project'.\"]" + ) assert error_message in caplog.messages @@ -632,23 +636,24 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project @pytest.mark.parametrize( "status_code, response_errorMessages, expected_log_message", ( - (400, - ["The value 'incorrect_project' does not exist for the field 'project'."], - ( - "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." - " Please grant the user to the project. " - "Errors: [\"The value \'incorrect_project\' does not exist for the field \'project\'.\"]" - ) - ), + ( + 400, + ["The value 'incorrect_project' does not exist for the field 'project'."], ( - 403, - ["The value 'incorrect_project' doesn't have permission for the field 'project'."], - ( - 'Stream `issues`. An error occurred, details:' - ' Errors: ["The value \'incorrect_project\' doesn\'t have permission for the field \'project\'."]' - ) + "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." + " Please grant the user to the project. " + "Errors: [\"The value 'incorrect_project' does not exist for the field 'project'.\"]" ), - ) + ), + ( + 403, + ["The value 'incorrect_project' doesn't have permission for the field 'project'."], + ( + "Stream `issues`. An error occurred, details:" + " Errors: [\"The value 'incorrect_project' doesn't have permission for the field 'project'.\"]" + ), + ), + ), ) def test_python_issues_stream_skip_on_http_codes_error_handling(config, status_code, response_errorMessages, expected_log_message, caplog): responses.add( @@ -689,8 +694,7 @@ def test_python_issues_stream_updated_state(config): stream = Issues(**args) updated_state = stream._get_updated_state( - current_stream_state={"updated": "2021-01-01T00:00:00Z"}, - latest_record={"updated": "2021-01-02T00:00:00Z"} + current_stream_state={"updated": "2021-01-01T00:00:00Z"}, latest_record={"updated": "2021-01-02T00:00:00Z"} ) assert updated_state == {"updated": "2021-01-02T00:00:00Z"} @@ -703,7 +707,7 @@ def test_python_issues_stream_updated_state(config): ("pullrequest={dataType=pullrequest, state=thestate, stateCount=1}", True), ("pullrequest={dataType=pullrequest, state=thestate, stateCount=0}", False), ("{}", False), - ) + ), ) def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pull_request): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) @@ -721,7 +725,9 @@ def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pul @responses.activate -def test_python_pull_requests_stream_has_pull_request(config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter): +def test_python_pull_requests_stream_has_pull_request( + config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter +): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} issues_stream = Issues(**args) @@ -822,7 +828,11 @@ def test_project_permissions_stream(config, mock_non_deleted_projects_responses, @responses.activate def test_project_email_stream(config, mock_non_deleted_projects_responses, mock_project_emails): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -836,7 +846,11 @@ def test_project_components_stream(config, mock_non_deleted_projects_responses, json=project_components_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -850,7 +864,11 @@ def test_permissions_stream(config, permissions_response): json=permissions_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build(), + ) assert len(output.records) == 1 assert len(responses.calls) == 1 @@ -869,7 +887,9 @@ def test_labels_stream(config, labels_response): json={}, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build() + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -955,38 +975,33 @@ def test_project_versions_stream(config, mock_non_deleted_projects_responses, pr @pytest.mark.parametrize( "stream, expected_records_number, expected_calls_number, log_message", [ - ( - "issues", - 2, - 4, - "The user doesn't have permission to the project. Please grant the user to the project." - ), + ("issues", 2, 4, "The user doesn't have permission to the project. Please grant the user to the project."), ( "issue_custom_field_contexts", 2, 4, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_contexts`. An error occurred, details: ['Not found issue custom field context for issue fields issuetype2']. Skipping for now. ", ), ( "issue_custom_field_options", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_options`. An error occurred, details: ['Not found issue custom field options for issue fields issuetype3']. Skipping for now. ", ), ( "issue_watchers", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_watchers`. An error occurred, details: ['Not found watchers for issue TESTKEY13-2']. Skipping for now. ", ), ( "project_email", 4, 4, - "Forbidden. You don't have permission to access this resource." + "Forbidden. You don't have permission to access this resource.", # "Stream `project_email`. An error occurred, details: ['No access to emails for project 3']. Skipping for now. ", ), ], @@ -1009,7 +1024,9 @@ def test_skip_slice( log_message, ): config["projects"] = config.get("projects", []) + ["Project3", "Project4"] - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build() + ) assert len(output.records) == expected_records_number assert len(responses.calls) == expected_calls_number diff --git a/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaviyo/main.py b/airbyte-integrations/connectors/source-klaviyo/main.py index 5b8c871c3d7d..bda84b0215c4 100644 --- a/airbyte-integrations/connectors/source-klaviyo/main.py +++ b/airbyte-integrations/connectors/source-klaviyo/main.py @@ -4,5 +4,6 @@ from source_klaviyo.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py index f7938ab9e6e7..c25c6a8c4983 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py @@ -5,11 +5,12 @@ import logging from typing import Dict, Optional +from requests import HTTPError, codes + from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING -from requests import HTTPError, codes class KlaviyoAvailabilityStrategy(HttpAvailabilityStrategy): diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py index c4088bf6c7b3..bd412653d7cf 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py @@ -7,6 +7,7 @@ import dpath import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py index 270f09704349..411cb965fc93 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py @@ -3,10 +3,11 @@ from typing import Optional, Union import requests +from requests.exceptions import InvalidURL + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, ResponseAction -from requests.exceptions import InvalidURL class KlaviyoErrorHandler(DefaultErrorHandler): diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py index 4607ff3dea19..54eda7f3b7cd 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py @@ -8,9 +8,10 @@ import traceback from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_klaviyo import SourceKlaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 1cff0bffa204..487d34bc4307 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -8,6 +8,8 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum +from requests import Response + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import WaitTimeFromHeaderBackoffStrategy from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy @@ -16,7 +18,6 @@ from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler, HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction -from requests import Response from .availability_strategy import KlaviyoAvailabilityStrategy from .exceptions import KlaviyoBackoffError @@ -128,7 +129,6 @@ def read_records( stream_slice: Optional[Mapping[str, Any]] = None, stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: - current_state = self.state or {} try: for record in super().read_records(sync_mode, cursor_field, stream_slice, current_state): @@ -263,7 +263,6 @@ def _set_campaign_message(self, record: Mapping[str, Any]) -> None: record["campaign_message"] = campaign_message_response.json().get("data") def get_error_handler(self) -> ErrorHandler: - error_mapping = DEFAULT_ERROR_MAPPING | { 404: ErrorResolution(ResponseAction.IGNORE, FailureType.config_error, "Resource not found. Ignoring.") } diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py index 878978270154..a217637afe7d 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py @@ -5,7 +5,7 @@ class KlaviyoConfigBuilder: def __init__(self) -> None: - self._config = {"api_key":"an_api_key","start_date":"2021-01-01T00:00:00Z"} + self._config = {"api_key": "an_api_key", "start_date": "2021-01-01T00:00:00Z"} def with_start_date(self, start_date: datetime) -> "KlaviyoConfigBuilder": self._config["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py index 969306e0d75c..2395a79040fa 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py @@ -3,6 +3,8 @@ from typing import Any, Dict, Optional from unittest import TestCase +from source_klaviyo import SourceKlaviyo + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -17,7 +19,7 @@ find_template, ) from integration.config import KlaviyoConfigBuilder -from source_klaviyo import SourceKlaviyo + _ENDPOINT_TEMPLATE_NAME = "profiles" _START_DATE = datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc) @@ -37,11 +39,11 @@ def _a_profile_request(start_date: datetime) -> HttpRequest: return HttpRequest( url=f"https://a.klaviyo.com/api/profiles", query_params={ - "additional-fields[profile]": "predictive_analytics", - "page[size]": "100", - "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", - "sort": "updated" - } + "additional-fields[profile]": "predictive_analytics", + "page[size]": "100", + "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", + "sort": "updated", + }, ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py index 211a158290e3..df84e66f8029 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py @@ -34,26 +34,26 @@ def extractor(mock_config, mock_field_path, mock_decoder): return KlaviyoIncludedFieldExtractor(mock_field_path, mock_config, mock_decoder) -@patch('dpath.get') -@patch('dpath.values') +@patch("dpath.get") +@patch("dpath.values") def test_extract_records_by_path(mock_values, mock_get, extractor, mock_response, mock_decoder): - mock_values.return_value = [{'key': 'value'}] - mock_get.return_value = {'key': 'value'} - mock_decoder.decode.return_value = {'data': 'value'} + mock_values.return_value = [{"key": "value"}] + mock_get.return_value = {"key": "value"} + mock_decoder.decode.return_value = {"data": "value"} - field_paths = ['data'] + field_paths = ["data"] records = list(extractor.extract_records_by_path(mock_response, field_paths)) - assert records == [{'key': 'value'}] + assert records == [{"key": "value"}] mock_values.return_value = [] mock_get.return_value = None - records = list(extractor.extract_records_by_path(mock_response, ['included'])) + records = list(extractor.extract_records_by_path(mock_response, ["included"])) assert records == [] def test_update_target_records_with_included(extractor): - target_records = [{'relationships': {'type1': {'data': {'id': 1}}}}] - included_records = [{'id': 1, 'type': 'type1', 'attributes': {'key': 'value'}}] + target_records = [{"relationships": {"type1": {"data": {"id": 1}}}}] + included_records = [{"id": 1, "type": "type1", "attributes": {"key": "value"}}] updated_records = list(extractor.update_target_records_with_included(target_records, included_records)) - assert updated_records[0]['relationships']['type1']['data'] == {'id': 1, 'key': 'value'} + assert updated_records[0]["relationships"]["type1"]["data"] == {"id": 1, "key": "value"} diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py index 052a93ae0f30..7cf3138d22f9 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py @@ -4,6 +4,8 @@ from unittest.mock import MagicMock +from source_klaviyo.components.per_partition_state_migration import PerPartitionToSingleStateMigration + from airbyte_cdk.sources.declarative.models import ( CustomRetriever, DatetimeBasedCursor, @@ -14,7 +16,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory -from source_klaviyo.components.per_partition_state_migration import PerPartitionToSingleStateMigration + factory = ModelToComponentFactory() @@ -26,14 +28,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): input_state = { "states": [ - { - "partition": {"parent_id": "13506132"}, - "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"} - }, - { - "partition": {"parent_id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, + {"partition": {"parent_id": "13506132"}, "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"}}, + {"partition": {"parent_id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] } @@ -61,14 +57,10 @@ def _migrator(): parent_key="{{ parameters['parent_key_id'] }}", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ) - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py index a25db137a57c..17cc35eb6352 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py @@ -7,11 +7,13 @@ import pendulum import pytest -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from integration.config import KlaviyoConfigBuilder from source_klaviyo.source import SourceKlaviyo +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + logger = logging.getLogger("airbyte") @@ -29,16 +31,12 @@ def _source() -> SourceKlaviyo: ( 400, False, - ( - "Bad request. Please check your request parameters." - ), + ("Bad request. Please check your request parameters."), ), ( 403, False, - ( - "Please provide a valid API key and make sure it has permissions to read specified streams." - ), + ("Please provide a valid API key and make sure it has permissions to read specified streams."), ), ), ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py index 2ed50d5e46f4..f501f5aeaa68 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -14,11 +14,6 @@ import pendulum import pytest import requests -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from dateutil.relativedelta import relativedelta from integration.config import KlaviyoConfigBuilder from pydantic import BaseModel @@ -26,6 +21,13 @@ from source_klaviyo.source import SourceKlaviyo from source_klaviyo.streams import Campaigns, CampaignsDetailed, IncrementalKlaviyoStream, KlaviyoStream +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + _ANY_ATTEMPT_COUNT = 123 API_KEY = "some_key" START_DATE = pendulum.datetime(2020, 10, 10) @@ -36,6 +38,7 @@ EVENTS_STREAM_STATE_DATE = (datetime.fromisoformat(EVENTS_STREAM_CONFIG_START_DATE) + relativedelta(years=1)).isoformat() EVENTS_STREAM_TESTING_FREEZE_TIME = "2023-12-12 12:00:00" + def get_step_diff(provided_date: str) -> int: """ This function returns the difference in weeks between provided date and freeze time. @@ -44,6 +47,7 @@ def get_step_diff(provided_date: str) -> int: freeze_date = datetime.strptime(EVENTS_STREAM_TESTING_FREEZE_TIME, "%Y-%m-%d %H:%M:%S") return (freeze_date - provided_date).days // 7 + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceKlaviyo(CatalogBuilder().build(), KlaviyoConfigBuilder().build(), StateBuilder().build()) matches_by_name = [stream_config for stream_config in source.streams(config) if stream_config.name == stream_name] @@ -84,9 +88,7 @@ def path(self, **kwargs) -> str: class TestKlaviyoStream: def test_request_headers(self): stream = SomeStream(api_key=API_KEY) - expected_headers = { - "Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}" - } + expected_headers = {"Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}"} assert stream.request_headers() == expected_headers @pytest.mark.parametrize( @@ -148,9 +150,7 @@ def test_availability_strategy(self): "This is most likely due to insufficient permissions on the credentials in use. " "Try to create and use an API key with read permission for the 'some_stream' stream granted" ) - reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes( - stream, None, None, None - ) + reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes(stream, None, None, None) assert expected_status_code in reasons_for_unavailable_status_codes assert reasons_for_unavailable_status_codes[expected_status_code] == expected_message @@ -173,14 +173,11 @@ def test_backoff_time_large_retry_after(self): response_mock.headers = {"Retry-After": retry_after} with pytest.raises(AirbyteTracedException) as e: stream.get_backoff_strategy().backoff_time(response_mock, _ANY_ATTEMPT_COUNT) - error_message = ( - "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." - ) + error_message = "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." assert str(e.value) == error_message class TestIncrementalKlaviyoStream: - @staticmethod def generate_api_urls(start_date_str: str) -> list[(str, str)]: """ @@ -197,12 +194,12 @@ def generate_api_urls(start_date_str: str) -> list[(str, str)]: end_date = current_date start_date_str = start_date.strftime("%Y-%m-%dT%H:%M:%S") + start_date.strftime("%z") end_date_str = end_date.strftime("%Y-%m-%dT%H:%M:%S") + end_date.strftime("%z") - base_url = 'https://a.klaviyo.com/api/events' + base_url = "https://a.klaviyo.com/api/events" query_params = { - 'fields[metric]': 'name,created,updated,integration', - 'include': 'metric', - 'filter': f'greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})', - 'sort': 'datetime' + "fields[metric]": "name,created,updated,integration", + "include": "metric", + "filter": f"greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})", + "sort": "datetime", } encoded_query = urllib.parse.urlencode(query_params) encoded_url = f"{base_url}?{encoded_query}" @@ -289,7 +286,6 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso latest_record={stream.cursor_field: latest_cursor}, ) == {stream.cursor_field: expected_cursor} - @freezegun.freeze_time("2023-12-12 12:00:00") @pytest.mark.parametrize( # expected_amount_of_results: we put 1 record for every request @@ -297,20 +293,20 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso ( ( # we pick the state - EVENTS_STREAM_CONFIG_START_DATE, - EVENTS_STREAM_STATE_DATE, - get_step_diff(EVENTS_STREAM_STATE_DATE) + 1 # adding last request + EVENTS_STREAM_CONFIG_START_DATE, + EVENTS_STREAM_STATE_DATE, + get_step_diff(EVENTS_STREAM_STATE_DATE) + 1, # adding last request ), ( - # we pick the config start date - EVENTS_STREAM_CONFIG_START_DATE, - None, - get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1 # adding last request + # we pick the config start date + EVENTS_STREAM_CONFIG_START_DATE, + None, + get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1, # adding last request ), ( - "", - "", - get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1 # adding last request + "", + "", + get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1, # adding last request ), ), ) @@ -370,16 +366,13 @@ class TestSemiIncrementalKlaviyoStream: ) def test_read_records(self, start_date, stream_state, input_records, expected_records, requests_mock): stream = get_stream_by_name("metrics", CONFIG | {"start_date": start_date}) - requests_mock.register_uri( - "GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records} - ) + requests_mock.register_uri("GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records}) stream.stream_state = {stream.cursor_field: stream_state if stream_state else start_date} records = get_records(stream=stream, sync_mode=SyncMode.incremental) assert records == expected_records class TestProfilesStream: - def test_read_records(self, requests_mock): stream = get_stream_by_name("profiles", CONFIG) json = { @@ -617,9 +610,9 @@ def test_stream_slices(self): ) def test_request_params(self, stream_state, stream_slice, next_page_token, expected_params): stream = Campaigns(api_key=API_KEY) - assert stream.request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ) == expected_params + assert ( + stream.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) == expected_params + ) class TestCampaignsDetailedStream: @@ -647,7 +640,9 @@ def test_set_recipient_count_not_found(self, requests_mock): mocked_response.ok = False mocked_response.status_code = 404 mocked_response.json.return_value = {} - with patch.object(stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response)): + with patch.object( + stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response) + ): stream._set_recipient_count(record) assert record["estimated_recipient_count"] == 0 diff --git a/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kyriba/main.py b/airbyte-integrations/connectors/source-kyriba/main.py index cd0b8f1f2f3e..33236c5afe07 100644 --- a/airbyte-integrations/connectors/source-kyriba/main.py +++ b/airbyte-integrations/connectors/source-kyriba/main.py @@ -4,5 +4,6 @@ from source_kyriba.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py index cac3eb31f5cf..81155b78ea89 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py @@ -8,6 +8,7 @@ import backoff import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py index 58ff037abc67..9e53da81f4d2 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py @@ -20,10 +20,7 @@ def patch_base_class(mocker): def test_stream_slices(patch_base_class): stream = BankBalancesStream(**config()) - account_uuids = [ - {"account_uuid": "first"}, - {"account_uuid": "second"} - ] + account_uuids = [{"account_uuid": "first"}, {"account_uuid": "second"}] stream.get_account_uuids = MagicMock(return_value=account_uuids) stream.start_date = date(2022, 1, 1) stream.end_date = date(2022, 1, 2) @@ -43,7 +40,7 @@ def test_stream_slices(patch_base_class): { "account_uuid": "second", "date": "2022-01-02", - } + }, ] slices = stream.stream_slices() assert slices == expected diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py index ae0b83acd3a4..904b7e0564ba 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py @@ -7,10 +7,11 @@ from unittest.mock import MagicMock import requests -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyriba.source import CashFlows +from airbyte_cdk.models import SyncMode + from .test_streams import config diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py index 5ddcaa10b8d9..cd6223667e06 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyriba.source import IncrementalKyribaStream +from airbyte_cdk.models import SyncMode + from .test_streams import config diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py index 1bda3981cbd6..6be7e9b33a63 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py @@ -6,6 +6,7 @@ from source_kyriba.source import KyribaClient, SourceKyriba + config = { "username": "username", "password": "password", @@ -13,6 +14,7 @@ "start_date": "2022-01-01", } + def test_check_connection(mocker): source = SourceKyriba() KyribaClient.login = MagicMock() diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py index 85f363267d8b..99d6da8a1a89 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py @@ -7,9 +7,10 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_kyriba.source import KyribaClient, KyribaStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.fixture def patch_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py index 77f078a20c15..6b6724504678 100644 --- a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kyve/main.py b/airbyte-integrations/connectors/source-kyve/main.py index a3740b34d958..cd5ae9e0c5f3 100644 --- a/airbyte-integrations/connectors/source-kyve/main.py +++ b/airbyte-integrations/connectors/source-kyve/main.py @@ -4,5 +4,6 @@ from source_kyve.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py index 2ec43d5c80db..d9f7ba7806d7 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -39,7 +40,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: pools = config.get("pool_ids").split(",") start_ids = config.get("start_ids").split(",") - for (pool_id, start_id) in zip(pools, start_ids): + for pool_id, start_id in zip(pools, start_ids): response = requests.get(f"{config['url_base']}/kyve/query/v1beta1/pool/{pool_id}") pool_data = response.json().get("pool").get("data") diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py index 6688a5832f2f..d2096615d53a 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py @@ -8,10 +8,12 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream from source_kyve.utils import query_endpoint + logger = logging.getLogger("airbyte") # 1: Arweave diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py index ea824656df58..1a87d748b08f 100644 --- a/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyve.source import KYVEStream as IncrementalKyveStream +from airbyte_cdk.models import SyncMode + from . import config, pool_data diff --git a/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/main.py b/airbyte-integrations/connectors/source-linkedin-ads/main.py index 899a7e8614a4..bf3f38b6d9fc 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/main.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/main.py @@ -4,5 +4,6 @@ from source_linkedin_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py index 2fcb05b7f7fb..1ecc55bcccba 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py @@ -12,6 +12,8 @@ import pendulum import requests +from isodate import Duration, parse_duration + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import CursorFactory, DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString @@ -31,7 +33,6 @@ from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS -from isodate import Duration, parse_duration from .utils import ANALYTICS_FIELDS_V2, FIELDS_CHUNK_SIZE, transform_data diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py index 276afed6da58..5bfbe9788132 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py @@ -10,6 +10,7 @@ from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 6813aaf34da1..3933f7dca212 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -14,6 +14,7 @@ from .utils import update_specific_key + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index b10d822ae9ca..5237fc476f4b 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -7,6 +7,7 @@ import pendulum as pdm + # replace `pivot` with `_pivot`, to allow redshift normalization, # since `pivot` is a reserved keyword for Destination Redshift, # on behalf of https://github.com/airbytehq/airbyte/issues/13018, diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index d93575984386..69255efc6a07 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -8,6 +8,7 @@ from source_linkedin_ads.source import SourceLinkedinAds + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py index 2fa7fbf0dd34..3ac0fae77771 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py @@ -18,6 +18,7 @@ StreamSlice, ) + logger = logging.getLogger("airbyte") @@ -35,7 +36,13 @@ def mock_response(): @pytest.fixture def mock_analytics_cursor_params(): - return {"start_datetime": MagicMock(), "cursor_field": MagicMock(), "datetime_format": "%s", "config": MagicMock(), "parameters": MagicMock()} + return { + "start_datetime": MagicMock(), + "cursor_field": MagicMock(), + "datetime_format": "%s", + "config": MagicMock(), + "parameters": MagicMock(), + } @pytest.fixture diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py index 931da4bfbbcd..17ee2e8dde59 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -6,13 +6,15 @@ import pytest import requests +from conftest import find_stream +from source_linkedin_ads.source import SourceLinkedinAds + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator -from conftest import find_stream -from source_linkedin_ads.source import SourceLinkedinAds + logger = logging.getLogger("airbyte") @@ -90,7 +92,9 @@ def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> No @pytest.mark.parametrize("error_code", [429, 500, 503]) def test_should_retry_on_error(self, error_code, requests_mock, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) stream = find_stream("accounts", TEST_CONFIG) requests_mock.register_uri( @@ -102,11 +106,10 @@ def test_should_retry_on_error(self, error_code, requests_mock, mocker): def test_custom_streams(self): config = {"ad_analytics_reports": [{"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}], **TEST_CONFIG} - ad_campaign_analytics = find_stream('ad_campaign_analytics', config) + ad_campaign_analytics = find_stream("ad_campaign_analytics", config) for stream in self._instance._create_custom_ad_analytics_streams(config=config): assert isinstance(stream, type(ad_campaign_analytics)) - @pytest.mark.parametrize( "stream_name, expected", [ @@ -136,26 +139,23 @@ def test_path(self, stream_name, expected): @pytest.mark.parametrize( ("status_code", "is_connection_successful", "error_msg"), ( - ( - 400, - False, - ( - "Bad request. Please check your request parameters." - ), - ), - ( - 403, - False, - ( - "Forbidden. You don't have permission to access this resource." - ), - ), - (200, True, None), + ( + 400, + False, + ("Bad request. Please check your request parameters."), + ), + ( + 403, + False, + ("Forbidden. You don't have permission to access this resource."), + ), + (200, True, None), ), ) def test_check_connection(self, requests_mock, status_code, is_connection_successful, error_msg, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", - side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) json = {"elements": [{"data": []}] * 500} if 200 >= status_code < 300 else {} requests_mock.register_uri( diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py index 197f43ea8e64..01c828362c8f 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py @@ -6,11 +6,13 @@ import os from typing import Any, Mapping -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from conftest import find_stream from freezegun import freeze_time +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + + # Test input arguments for the `make_analytics_slices` TEST_KEY_VALUE_MAP = {"camp_id": "id"} TEST_START_DATE = "2021-08-01" @@ -50,7 +52,8 @@ def test_read_records(requests_mock): requests_mock.get("https://api.linkedin.com/rest/adAccounts", json={"elements": [{"id": 1}]}) requests_mock.get( "https://api.linkedin.com/rest/adAccounts/1/adCampaigns?q=search&search=(status:(values:List(ACTIVE,PAUSED,ARCHIVED,COMPLETED,CANCELED,DRAFT,PENDING_DELETION,REMOVED)))", - json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}) + json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}, + ) requests_mock.get( "https://api.linkedin.com/rest/adAnalytics", [ diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py index e116afe2bbbd..83ca7aaf2547 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -2,7 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -""" This is the example of input record for the test_tranform_data. """ +"""This is the example of input record for the test_tranform_data.""" + input_test_data = [ { "targetingCriteria": { @@ -65,7 +66,7 @@ } }, "pivot": "TEST_PIVOT_VALUE", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] @@ -144,6 +145,6 @@ "end_date": "2021-08-13", "_pivot": "TEST_PIVOT_VALUE", "string_of_pivot_values": "TEST_PIVOT_VALUE_1,TEST_PIVOT_VALUE_2", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py index 73be5f6a0564..c62676cfc0ff 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py @@ -74,8 +74,8 @@ "nested_dictionary_update", "list_of_dictionaries_update", "excluded_key_in_nested_dict", - "nested_list_with_mixed_types" - ] + "nested_list_with_mixed_types", + ], ) def test_update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys, expected_output): result = update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linnworks/main.py b/airbyte-integrations/connectors/source-linnworks/main.py index ee964c061ce0..e6b07506ae4f 100644 --- a/airbyte-integrations/connectors/source-linnworks/main.py +++ b/airbyte-integrations/connectors/source-linnworks/main.py @@ -4,5 +4,6 @@ from source_linnworks.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 7192e2ec5826..32761a344de5 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -7,6 +7,7 @@ import pendulum import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator @@ -48,7 +49,6 @@ def get_auth_header(self) -> Mapping[str, Any]: return {"Authorization": self.get_access_token()} def get_access_token(self): - if self.token_has_expired(): t0 = pendulum.now() token, expires_in, server = self.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py index a3935d719844..fe7733524ec3 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py @@ -11,11 +11,12 @@ import pendulum import requests import vcr -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from requests.auth import AuthBase from vcr.cassette import Cassette +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator + class LinnworksStream(HttpStream, ABC): http_method = "POST" diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py index d0c1081e0625..1ab8760bc505 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py @@ -11,9 +11,10 @@ import pytest import requests import vcr +from source_linnworks.streams import IncrementalLinnworksStream, ProcessedOrderDetails, ProcessedOrders + from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.streams.http.http import HttpSubStream -from source_linnworks.streams import IncrementalLinnworksStream, ProcessedOrderDetails, ProcessedOrders @pytest.fixture diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py index 88fa5a9d4eaa..70e25009b6b4 100644 --- a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py @@ -7,6 +7,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-looker/main.py b/airbyte-integrations/connectors/source-looker/main.py index b6164cb0322b..cb6f8adb314b 100644 --- a/airbyte-integrations/connectors/source-looker/main.py +++ b/airbyte-integrations/connectors/source-looker/main.py @@ -4,5 +4,6 @@ from source_looker.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-looker/source_looker/components.py b/airbyte-integrations/connectors/source-looker/source_looker/components.py index d30b120a5e97..60352fffd197 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/components.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/components.py @@ -7,14 +7,15 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth + API_VERSION = "4.0" @dataclass class LookerAuthenticator(NoAuth): - """ Authenticator that sets the Authorization header on the HTTP requests sent using access token which is updated upon expiration. diff --git a/airbyte-integrations/connectors/source-looker/source_looker/source.py b/airbyte-integrations/connectors/source-looker/source_looker/source.py index 5a6e0487f5bb..7799272df347 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/source.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.core import Stream + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailchimp/main.py b/airbyte-integrations/connectors/source-mailchimp/main.py index c61875fb7a72..0145a6e026ef 100644 --- a/airbyte-integrations/connectors/source-mailchimp/main.py +++ b/airbyte-integrations/connectors/source-mailchimp/main.py @@ -4,5 +4,6 @@ from source_mailchimp.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py index 6df59797f6ae..f853ec6a4687 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py @@ -3,6 +3,7 @@ from typing import Any, Iterable, Mapping import requests + from airbyte_cdk.sources.declarative.extractors import DpathExtractor diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py index 59d0de0e9869..4952fac913eb 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py @@ -7,12 +7,14 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import FailureType from airbyte_cdk.sources import Source from airbyte_cdk.utils import AirbyteTracedException + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py index 602562aa2db4..1b37739b4ab4 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py @@ -4,16 +4,18 @@ from unittest import TestCase import freezegun +from source_mailchimp import SourceMailchimp + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from source_mailchimp import SourceMailchimp from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_start_date(datetime.datetime(2023, 1, 1, 0, 0, 0, 1000)).build() diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py index 7f9ad40d0c7f..ec7029480f6a 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py @@ -6,9 +6,10 @@ import json import requests -from airbyte_cdk.sources.declarative.decoders import JsonDecoder from source_mailchimp.components import MailChimpRecordExtractorEmailActivity +from airbyte_cdk.sources.declarative.decoders import JsonDecoder + def test_email_activity_extractor(): decoder = JsonDecoder(parameters={}) diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py index 20fe8312352e..b6d0e41de2c2 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py @@ -6,10 +6,12 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from source_mailchimp import SourceMailchimp from source_mailchimp.config_migrations import MigrateDataCenter +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + + # BASE ARGS SOURCE: YamlDeclarativeSource = SourceMailchimp() diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-marketo/main.py b/airbyte-integrations/connectors/source-marketo/main.py index 4b7b8e8d1708..0b087d968a27 100644 --- a/airbyte-integrations/connectors/source-marketo/main.py +++ b/airbyte-integrations/connectors/source-marketo/main.py @@ -4,5 +4,6 @@ from source_marketo.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 418c327cc104..1eee453d1740 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -11,6 +11,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py b/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py index 9926e2e2084e..426d437b66a2 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py @@ -5,6 +5,7 @@ from datetime import datetime + STRING_TYPES = [ "string", "email", diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py index f088ce69b9fc..b4f03f750bd4 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py @@ -9,9 +9,11 @@ import pendulum import pytest -from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from source_marketo.source import Activities, MarketoAuthenticator, SourceMarketo +from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream + + START_DATE = pendulum.now().subtract(days=75) @@ -77,7 +79,10 @@ def _generator(min_size: int): def fake_records_gen(): new_line = "\n" for i in range(1000): - yield f"{str(faker.random_int())},{faker.random_int()},{faker.date_of_birth()},{faker.random_int()}," f"{faker.random_int()},{faker.email()},{faker.postcode()}{new_line}" + yield ( + f"{str(faker.random_int())},{faker.random_int()},{faker.date_of_birth()},{faker.random_int()}," + f"{faker.random_int()},{faker.email()},{faker.postcode()}{new_line}" + ) size, records = 0, 0 path = os.path.realpath(str(time.time())) @@ -98,9 +103,7 @@ def fake_records_gen(): def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> DeclarativeStream: source = SourceMarketo() - matches_by_name = [ - stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name - ] + matches_by_name = [stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name] if not matches_by_name: raise ValueError("Please provide a valid stream name.") return matches_by_name[0] diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index d77625438bbc..cb5d3988aa85 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -11,13 +11,15 @@ import pendulum import pytest import requests +from source_marketo.source import Activities, IncrementalMarketoStream, Leads, MarketoExportCreate, MarketoStream, SourceMarketo + from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.utils import AirbyteTracedException -from source_marketo.source import Activities, IncrementalMarketoStream, Leads, MarketoExportCreate, MarketoStream, SourceMarketo from .conftest import START_DATE, get_stream_by_name + logger = logging.getLogger("airbyte") @@ -38,12 +40,7 @@ def test_should_retry_quota_exceeded(config, requests_mock): response_json = { "requestId": "d2ca#18c0b9833bf", "success": False, - "errors": [ - { - "code": "1029", - "message": "Export daily quota 500MB exceeded." - } - ] + "errors": [{"code": "1029", "message": "Export daily quota 500MB exceeded."}], } requests_mock.register_uri("GET", create_job_url, status_code=200, json=response_json) @@ -132,8 +129,8 @@ def test_activities_schema(activity, expected_schema, config): ( ( "Campaign Run ID,Choice Number,Has Predictive,Step ID,Test Variant,attributes\n" - "1,3,true,10,15,{\"spam\": \"true\"}\n" - "2,3,false,11,16,{\"spam\": \"false\"}" + '1,3,true,10,15,{"spam": "true"}\n' + '2,3,false,11,16,{"spam": "false"}' ), [ { @@ -231,9 +228,7 @@ def test_parse_response_incremental(config, requests_mock): created_at_record_1 = START_DATE.add(days=1).strftime("%Y-%m-%dT%H:%M:%SZ") created_at_record_2 = START_DATE.add(days=3).strftime("%Y-%m-%dT%H:%M:%SZ") current_state = START_DATE.add(days=2).strftime("%Y-%m-%dT%H:%M:%SZ") - response = { - "result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}] - } + response = {"result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}]} requests_mock.get("/rest/v1/campaigns.json", json=response) stream = get_stream_by_name("campaigns", config) @@ -322,16 +317,8 @@ def test_get_updated_state(config, latest_record, current_state, expected_state) def test_filter_null_bytes(config): stream = Leads(config) - test_lines = [ - "Hello\x00World\n", - "Name,Email\n", - "John\x00Doe,john.doe@example.com\n" - ] - expected_lines = [ - "HelloWorld\n", - "Name,Email\n", - "JohnDoe,john.doe@example.com\n" - ] + test_lines = ["Hello\x00World\n", "Name,Email\n", "John\x00Doe,john.doe@example.com\n"] + expected_lines = ["HelloWorld\n", "Name,Email\n", "JohnDoe,john.doe@example.com\n"] filtered_lines = stream.filter_null_bytes(test_lines) for expected_line, filtered_line in zip(expected_lines, filtered_lines): assert expected_line == filtered_line @@ -340,15 +327,8 @@ def test_filter_null_bytes(config): def test_csv_rows(config): stream = Leads(config) - test_lines = [ - "Name,Email\n", - "John Doe,john.doe@example.com\n", - "Jane Doe,jane.doe@example.com\n" - ] - expected_records = [ - {"Name": "John Doe", "Email": "john.doe@example.com"}, - {"Name": "Jane Doe", "Email": "jane.doe@example.com"} - ] + test_lines = ["Name,Email\n", "John Doe,john.doe@example.com\n", "Jane Doe,jane.doe@example.com\n"] + expected_records = [{"Name": "John Doe", "Email": "john.doe@example.com"}, {"Name": "Jane Doe", "Email": "jane.doe@example.com"}] records = stream.csv_rows(test_lines) for expected_record, record in zip(expected_records, records): assert expected_record == record diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py index 453954ab3641..38cff9225bdd 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py @@ -8,6 +8,7 @@ import pytest from source_marketo.utils import clean_string, format_value, to_datetime_str + test_data = [ (1, {"type": "integer"}, int), ("string", {"type": "string"}, str), diff --git a/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py index 88b4cf3808e8..18cbfd717811 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py @@ -4,5 +4,6 @@ from source_microsoft_dataverse.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py index a3a14fc8addb..8f33e6eaf55d 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py @@ -6,6 +6,7 @@ from typing import Any, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import Oauth2Authenticator @@ -25,7 +26,6 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: class AirbyteType(Enum): - String = {"type": ["null", "string"]} Boolean = {"type": ["null", "boolean"]} Timestamp = {"type": ["null", "string"], "format": "date-time", "airbyte_type": "timestamp_with_timezone"} @@ -34,7 +34,6 @@ class AirbyteType(Enum): class DataverseType(Enum): - String = AirbyteType.String Uniqueidentifier = AirbyteType.String DateTime = AirbyteType.Timestamp diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py index a85426a7c098..3f384d8e31b8 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py @@ -8,13 +8,13 @@ from urllib import parse import requests + from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream # Basic full refresh stream class MicrosoftDataverseStream(HttpStream, ABC): - # Base url will be set by init(), using information provided by the user through config input url_base = "" primary_key = "" @@ -97,7 +97,6 @@ def path( # Basic incremental stream class IncrementalMicrosoftDataverseStream(MicrosoftDataverseStream, IncrementalMixin, ABC): - delta_token_field = "$deltatoken" state_checkpoint_interval = None # For now we just use the change tracking as state, and it is only emitted on last page diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py index 0e93f16521ab..402667e9e3fd 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py @@ -6,11 +6,12 @@ from unittest import mock from unittest.mock import MagicMock -from airbyte_cdk.models import SyncMode from source_microsoft_dataverse.dataverse import AirbyteType from source_microsoft_dataverse.source import SourceMicrosoftDataverse from source_microsoft_dataverse.streams import IncrementalMicrosoftDataverseStream, MicrosoftDataverseStream +from airbyte_cdk.models import SyncMode + @mock.patch("source_microsoft_dataverse.source.do_request") def test_check_connection(mock_request): diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py index f12c7112ca11..4d33342920aa 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py @@ -5,5 +5,6 @@ from source_microsoft_onedrive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py index b5bc8890dacc..a94c5f3d02db 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from pydantic import BaseModel, Field +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec + class OAuthCredentials(BaseModel): """ diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py index b7c13534a3e5..2623fff2842f 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py @@ -11,12 +11,13 @@ import requests import smart_open -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from msal import ConfidentialClientApplication from msal.exceptions import MsalServiceError from office365.graph_client import GraphClient + +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py index 4823f2652049..8c2d6ede5243 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py @@ -4,5 +4,6 @@ from source_microsoft_sharepoint.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py index e1053a323696..c61196de4e47 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from pydantic.v1 import BaseModel, Field +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec + class OAuthCredentials(BaseModel): """ diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py index 8db55c5c6609..baab6f86505b 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py @@ -11,11 +11,12 @@ import requests import smart_open +from msal import ConfidentialClientApplication +from office365.graph_client import GraphClient + from airbyte_cdk import AirbyteTracedException, FailureType from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from msal import ConfidentialClientApplication -from office365.graph_client import GraphClient from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec from .utils import FolderNotFoundException, MicrosoftSharePointRemoteFile, execute_query_with_retry, filter_http_urls diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py index 7a658c1b2431..a1741915978b 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py @@ -8,6 +8,7 @@ from airbyte_cdk import AirbyteTracedException, FailureType from airbyte_cdk.sources.file_based.remote_file import RemoteFile + LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py index 205e1399c52c..ee2d8a3e78b0 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock, Mock, PropertyMock, call, patch import pytest -from airbyte_cdk import AirbyteTracedException from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec from source_microsoft_sharepoint.stream_reader import ( FileReadMode, @@ -17,6 +16,8 @@ ) from wcmatch.glob import GLOBSTAR, globmatch +from airbyte_cdk import AirbyteTracedException + def create_mock_drive_item(is_file, name, children=None): """Helper function to create a mock drive item.""" @@ -465,9 +466,10 @@ def test_get_shared_drive_object( ], ) def test_drives_property(auth_type, user_principal_name, has_refresh_token): - with patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, patch( - "source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client" - ) as mock_one_drive_client: + with ( + patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, + patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client") as mock_one_drive_client, + ): refresh_token = "dummy_refresh_token" if has_refresh_token else None # Setup for different authentication types config_mock = MagicMock( diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py index 5e88631a341f..f2eae51b2c76 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py @@ -6,9 +6,10 @@ from unittest.mock import Mock, patch import pytest -from airbyte_cdk import AirbyteTracedException from source_microsoft_sharepoint.utils import execute_query_with_retry, filter_http_urls +from airbyte_cdk import AirbyteTracedException + class MockResponse: def __init__(self, status_code, headers=None): @@ -49,9 +50,10 @@ def test_execute_query_with_retry(status_code, retry_after_header, expected_retr obj = Mock() obj.execute_query = Mock(side_effect=MockException(status_code, {"Retry-After": retry_after_header})) - with patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, patch( - "source_microsoft_sharepoint.utils.datetime" - ) as mock_datetime: + with ( + patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, + patch("source_microsoft_sharepoint.utils.datetime") as mock_datetime, + ): start_time = datetime(2021, 1, 1, 0, 0, 0) if retry_after_header: mock_datetime.now.side_effect = [start_time] * 2 + [ diff --git a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mixpanel/main.py b/airbyte-integrations/connectors/source-mixpanel/main.py index df8cb33fc826..b08271eca1ab 100644 --- a/airbyte-integrations/connectors/source-mixpanel/main.py +++ b/airbyte-integrations/connectors/source-mixpanel/main.py @@ -4,5 +4,6 @@ from source_mixpanel.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py index 757909d1f8dc..838f3c1e1709 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py @@ -5,6 +5,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py index 1730f7673647..9216c0cf7eaa 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py @@ -6,6 +6,7 @@ import dpath.util import requests + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString @@ -32,7 +33,6 @@ def get_request_headers( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return {"Accept": "application/json"} def get_request_params( @@ -62,7 +62,6 @@ def _request_params( return super()._request_params(stream_state, stream_slice, next_page_token, extra_params) def send_request(self, **kwargs) -> Optional[requests.Response]: - if self.reqs_per_hour_limit: if self.is_first_request: self.is_first_request = False diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py index 628cd46dcbda..06b61176db62 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py @@ -10,6 +10,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py index a5d99f1566ae..815e24e61fa6 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py @@ -6,6 +6,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py index 4ed3559f1af5..a47c2611854e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 48a4184fbf42..1e5dccab2eeb 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import pendulum + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 2d633492ddb1..c59258305abd 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -9,11 +9,12 @@ import pendulum import requests +from pendulum import Date +from requests.auth import AuthBase + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler -from pendulum import Date -from requests.auth import AuthBase from source_mixpanel.backoff_strategy import MixpanelStreamBackoffStrategy from source_mixpanel.errors_handlers import MixpanelStreamErrorHandler from source_mixpanel.utils import fix_date_time diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py index e643b5836d43..ed3d202c98e7 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py index 69c842e5e255..1df5c3b847c1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py @@ -45,12 +45,14 @@ def patch_time(mocker): ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH + def delete_cache_files(cache_directory): directory_path = Path(cache_directory) if directory_path.exists() and directory_path.is_dir(): for file_path in directory_path.glob("*.sqlite"): file_path.unlink() + @pytest.fixture(autouse=True) def clear_cache_before_each_test(): # The problem: Once the first request is cached, we will keep getting the cached result no matter what setup we prepared for a particular test. diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py index a2132148baa2..57da922485ca 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py @@ -5,10 +5,12 @@ from unittest.mock import patch import pytest -from airbyte_cdk.entrypoint import AirbyteEntrypoint from source_mixpanel.config_migrations import MigrateProjectId from source_mixpanel.source import SourceMixpanel +from airbyte_cdk.entrypoint import AirbyteEntrypoint + + # Test data for parametrized test test_data = [ # Test when only api_secret is present diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py index e1636caaef47..6a93ad167471 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py @@ -7,12 +7,14 @@ that will conflict in further data normalization, like: `userName` and `username` """ + from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from source_mixpanel.streams import Export +from airbyte_cdk.models import SyncMode + from .utils import get_url_to_mock, setup_response diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index f877bdd99779..4471eaa511b3 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -6,12 +6,14 @@ import logging import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel.source import SourceMixpanel, TokenAuthenticatorBase64 from source_mixpanel.streams import Export +from airbyte_cdk.utils import AirbyteTracedException + from .utils import command_check, get_url_to_mock, setup_response + logger = logging.getLogger("airbyte") @@ -22,12 +24,7 @@ def check_connection_url(config): return get_url_to_mock(export_stream) -@pytest.mark.parametrize( - "response_code,expect_success,response_json", - [ - (400, False, {"error": "Request error"}) - ] -) +@pytest.mark.parametrize("response_code,expect_success,response_json", [(400, False, {"error": "Request error"})]) def test_check_connection(requests_mock, check_connection_url, config_raw, response_code, expect_success, response_json): # requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, response_json)) requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=response_code, json=response_json) @@ -135,7 +132,7 @@ def test_streams_string_date(requests_mock, config_raw): "select_properties_by_default": True, "region": "EU", "date_window_size": 10, - "page_size": 1000 + "page_size": 1000, }, True, None, @@ -143,9 +140,9 @@ def test_streams_string_date(requests_mock, config_raw): ), ) def test_config_validation(config, success, expected_error_message, requests_mock): - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) try: is_success, message = SourceMixpanel().check_connection(None, config) except AirbyteTracedException as e: diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index 6f64dbf39aa5..9774037a6038 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -9,15 +9,17 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel import SourceMixpanel from source_mixpanel.streams import EngageSchema, Export, ExportSchema, IncrementalMixpanelStream, MixpanelStream from source_mixpanel.utils import read_full_refresh +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.utils import AirbyteTracedException + from .utils import get_url_to_mock, read_incremental, setup_response + logger = logging.getLogger("airbyte") MIXPANEL_BASE_URL = "https://mixpanel.com/api/2.0/" @@ -95,7 +97,7 @@ def cohorts_response(): ) -def init_stream(name='', config=None): +def init_stream(name="", config=None): streams = SourceMixpanel().streams(config) for stream in streams: if stream.name == name: @@ -104,10 +106,10 @@ def init_stream(name='', config=None): def test_cohorts_stream_incremental(requests_mock, cohorts_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - config_raw['start_date'] = '2022-01-01T00:00:00Z' + config_raw["start_date"] = "2022-01-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response) - cohorts_stream = init_stream('cohorts', config=config_raw) + cohorts_stream = init_stream("cohorts", config=config_raw) records = read_incremental(cohorts_stream, stream_state={"created": "2022-04-19 23:22:01"}, cursor_field=["created"]) @@ -158,25 +160,14 @@ def engage_response(): def test_engage_stream_incremental(requests_mock, engage_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2022-02-01T00:00:00Z' - config_raw['end_date'] = '2024-05-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2022-02-01T00:00:00Z" + config_raw["end_date"] = "2024-05-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) stream_state = {"last_seen": "2024-02-11T11:20:47"} records = list(read_incremental(stream, stream_state=stream_state, cursor_field=["last_seen"])) @@ -193,97 +184,97 @@ def test_engage_stream_incremental(requests_mock, engage_response, config_raw): {}, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "abnormal_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 0, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "medium_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 1, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "early_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "state_for_different_partition", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, }, { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, - } + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, + }, ] - } + }, ), ), ) @@ -291,26 +282,17 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi """Cohort_members stream has legacy state but actually it should always return all records because members in cohorts can be updated at any time """ - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{'id': 1111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}]) + requests_mock.register_uri( + "GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{"id": 1111, "name": "bla", "created": "2024-02-02T00:00:00Z"}] + ) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('cohort_members', config=config_raw) + stream = init_stream("cohort_members", config=config_raw) records = list(read_incremental(stream, stream_state=state, cursor_field=["last_seen"])) @@ -321,99 +303,85 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi def test_cohort_members_stream_pagination(requests_mock, engage_response, config_raw): """Cohort_members pagination""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' - - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[ - {'id': 71000, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 71111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}, - {'id': 72222, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 73333, "name":'bla', 'created': '2024-02-03T00:00:00Z'}, - ]) - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage", [ - { # initial request for 71000 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - }, - { # initial request for 71111 cohort and further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 2002, - "results": [ - { - "$distinct_id": "71111_1", - "$properties": { - "$created": "2024-03-01T11:20:47", - "$last_seen": "2024-03-01T11:20:47", + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" + requests_mock.register_uri( + "GET", + MIXPANEL_BASE_URL + "cohorts/list", + json=[ + {"id": 71000, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 71111, "name": "bla", "created": "2024-02-02T00:00:00Z"}, + {"id": 72222, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 73333, "name": "bla", "created": "2024-02-03T00:00:00Z"}, + ], + ) + requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage", + [ + { # initial request for 71000 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + { # initial request for 71111 cohort and further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 2002, + "results": [ + { + "$distinct_id": "71111_1", + "$properties": { + "$created": "2024-03-01T11:20:47", + "$last_seen": "2024-03-01T11:20:47", + }, }, - }, - { - "$distinct_id": "71111_2", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", - } - } - ] - } - }, { # initial request for 72222 cohort without further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 1, - "results": [ - { - "$distinct_id": "72222_1", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", + { + "$distinct_id": "71111_2", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, + }, + ], + }, + }, + { # initial request for 72222 cohort without further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 1, + "results": [ + { + "$distinct_id": "72222_1", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, } - } - ] - } - },{ # initial request for 73333 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - } - ] + ], + }, + }, + { # initial request for 73333 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + ], ) # request for 1 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", + json={ "page": 1, "session_id": "1234567890", "status": "ok", @@ -423,13 +391,16 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) # request for 2 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", + json={ "page": 2, "session_id": "1234567890", "status": "ok", @@ -439,27 +410,23 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) - stream = init_stream('cohort_members', config=config_raw) - + stream = init_stream("cohort_members", config=config_raw) + records = list(read_incremental(stream, stream_state={}, cursor_field=["last_seen"])) assert len(records) == 5 new_updated_state = stream.get_updated_state(current_stream_state={}, latest_record=records[-1] if records else None) - assert new_updated_state == {'states': [ - { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 71111, 'parent_slice': {}} - }, - { - 'cursor': {'last_seen': '2024-02-01T11:20:47'}, - 'partition': {'id': 72222, 'parent_slice': {}} - } - ]} + assert new_updated_state == { + "states": [ + {"cursor": {"last_seen": "2024-03-01T11:20:47"}, "partition": {"id": 71111, "parent_slice": {}}}, + {"cursor": {"last_seen": "2024-02-01T11:20:47"}, "partition": {"id": 72222, "parent_slice": {}}}, + ] + } @pytest.fixture @@ -493,37 +460,30 @@ def funnels_response(start_date): }, ) + @pytest.fixture def funnel_ids_response(start_date): - return setup_response( - 200, - [{ - "funnel_id": 36152117, - "name": "test" - }] - ) + return setup_response(200, [{"funnel_id": 36152117, "name": "test"}]) def test_funnels_stream(requests_mock, config, funnels_response, funnel_ids_response, config_raw): config_raw["start_date"] = "2024-01-01T00:00:00Z" config_raw["end_date"] = "2024-04-01T00:00:00Z" - stream = init_stream('funnels', config=config_raw) + stream = init_stream("funnels", config=config_raw) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels/list", funnel_ids_response) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels", funnels_response) stream_slices = list(stream.stream_slices(sync_mode=SyncMode.incremental)) assert len(stream_slices) > 3 - assert { - "funnel_id": stream_slices[0]['funnel_id'], - "name": stream_slices[0]['funnel_name'] - } == { + assert {"funnel_id": stream_slices[0]["funnel_id"], "name": stream_slices[0]["funnel_name"]} == { "funnel_id": "36152117", - "name": "test" + "name": "test", } records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0]) records = list(records) assert len(records) == 2 + @pytest.fixture def engage_schema_response(): return setup_response( @@ -552,7 +512,7 @@ def _minimize_schema(fill_schema, schema_original): def test_engage_schema(requests_mock, engage_schema_response, config_raw): - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) requests_mock.register_uri("GET", get_url_to_mock(EngageSchema(authenticator=MagicMock(), **config_raw)), engage_schema_response) type_schema = {} _minimize_schema(type_schema, stream.get_json_schema()) @@ -600,7 +560,7 @@ def test_update_engage_schema(requests_mock, config, config_raw): }, ), ) - engage_stream = init_stream('engage', config=config_raw) + engage_stream = init_stream("engage", config=config_raw) engage_schema = engage_stream.get_json_schema() assert "someNewSchemaField" in engage_schema["properties"] @@ -619,13 +579,10 @@ def annotations_response(): def test_annotations_stream(requests_mock, annotations_response, config_raw): - stream = init_stream('annotations', config=config_raw) + stream = init_stream("annotations", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", annotations_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -648,14 +605,12 @@ def revenue_response(): "status": "ok", }, ) -def test_revenue_stream(requests_mock, revenue_response, config_raw): - stream = init_stream('revenue', config=config_raw) + +def test_revenue_stream(requests_mock, revenue_response, config_raw): + stream = init_stream("revenue", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", revenue_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -675,7 +630,6 @@ def export_schema_response(): def test_export_schema(requests_mock, export_schema_response, config): - stream = ExportSchema(authenticator=MagicMock(), **config) requests_mock.register_uri("GET", get_url_to_mock(stream), export_schema_response) @@ -684,14 +638,14 @@ def test_export_schema(requests_mock, export_schema_response, config): records_length = sum(1 for _ in records) assert records_length == 2 -def test_export_get_json_schema(requests_mock, export_schema_response, config): +def test_export_get_json_schema(requests_mock, export_schema_response, config): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", export_schema_response) stream = Export(authenticator=MagicMock(), **config) schema = stream.get_json_schema() - assert "DYNAMIC_FIELD" in schema['properties'] + assert "DYNAMIC_FIELD" in schema["properties"] @pytest.fixture @@ -717,7 +671,6 @@ def export_response(): def test_export_stream(requests_mock, export_response, config): - stream = Export(authenticator=MagicMock(), **config) requests_mock.register_uri("GET", get_url_to_mock(stream), export_response) @@ -728,8 +681,8 @@ def test_export_stream(requests_mock, export_response, config): records_length = sum(1 for _ in records) assert records_length == 1 -def test_export_stream_fail(requests_mock, export_response, config): +def test_export_stream_fail(requests_mock, export_response, config): stream = Export(authenticator=MagicMock(), **config) error_message = "" requests_mock.register_uri("GET", get_url_to_mock(stream), status_code=400, text="Unable to authenticate request") diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index 1762aa42c718..3eeff0eac9fd 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -9,7 +9,6 @@ def test_date_slices(): - now = pendulum.today(tz="US/Pacific").date() # test with stream_state diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py index 5b08cd789244..4c2903b76df9 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py @@ -35,7 +35,9 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, stream_instance.state = stream_state slices = stream_instance.stream_slices(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_state=stream_state) for slice in slices: - records = stream_instance.read_records(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state) + records = stream_instance.read_records( + sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state + ) for record in records: stream_state = stream_instance.get_updated_state(stream_state, record) res.append(record) diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-monday/main.py b/airbyte-integrations/connectors/source-monday/main.py index 14f4fa2d0439..68776ccfc1f5 100644 --- a/airbyte-integrations/connectors/source-monday/main.py +++ b/airbyte-integrations/connectors/source-monday/main.py @@ -4,5 +4,6 @@ from source_monday.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-monday/source_monday/components.py b/airbyte-integrations/connectors/source-monday/source_monday/components.py index 802d23aacc81..d79e8c8e0124 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/components.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, List, Mapping, Optional, Union import dpath.util + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.incremental import Cursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -14,6 +15,7 @@ from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.core import Stream + RequestInput = Union[str, Mapping[str, str]] diff --git a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py index 126839bdecc7..6b0d08427d93 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py @@ -10,12 +10,14 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config, Record + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py index a6276416d2e5..131f69786a8e 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement + # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-monday/source_monday/source.py b/airbyte-integrations/connectors/source-monday/source_monday/source.py index 45fab7ff246f..868e33c0a027 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/source.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py index 3dd017d476b7..b25b2c27ad98 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class MondayBaseRequestBuilder(MondayRequestBuilder): diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py index 779d64d80af7..2457b51faac0 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py @@ -19,4 +19,3 @@ def build(self, file_path: Optional[str] = None) -> HttpResponse: if not file_path: return HttpResponse(json.dumps(find_template(str(self._status_code), __file__)), self._status_code) return HttpResponse(json.dumps(find_template(str(file_path), __file__)), self._status_code) - diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py index eab0deb8d5b1..654e84f75192 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py @@ -3,12 +3,13 @@ import operator from typing import Any, Dict, List, Optional +from source_monday import SourceMonday + from airbyte_cdk.models import AirbyteMessage from airbyte_cdk.models import Level as LogLevel from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import SyncMode -from source_monday import SourceMonday def read_stream( diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py index 30571cbd43f7..4b0aab530fb4 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py @@ -7,13 +7,14 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.models import AirbyteMessage, SyncMode, Type -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.streams import Stream from requests import Response from source_monday.components import IncrementalSingleSlice, IncrementalSubstreamSlicer from source_monday.extractor import MondayIncrementalItemsExtractor +from airbyte_cdk.models import AirbyteMessage, SyncMode, Type +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.streams import Stream + def _create_response(content: Any) -> Response: response = Response() @@ -144,7 +145,6 @@ def mock_parent_stream_slices(*args, **kwargs): ids=["no stream state", "successfully read parent record", "skip non_record AirbyteMessage"], ) def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, expected_slices): - slicer = IncrementalSubstreamSlicer( config={}, parameters={}, @@ -162,7 +162,6 @@ def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, ex def test_set_initial_state(): - slicer = IncrementalSubstreamSlicer( config={}, parameters={}, diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index 2037f13ee02a..fcd494ecb348 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -5,10 +5,12 @@ from unittest.mock import MagicMock import pytest +from source_monday import MondayGraphqlRequester + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader -from source_monday import MondayGraphqlRequester + nested_object_schema = { "root": { @@ -126,7 +128,6 @@ def test_get_schema_root_properties(mocker, monday_requester): def test_build_activity_query(mocker, monday_requester): - mock_stream_state = {"updated_at_int": 1636738688} object_arguments = {"stream_state": mock_stream_state} mocker.patch.object(MondayGraphqlRequester, "_get_object_arguments", return_value="stream_state:{{ stream_state['updated_at_int'] }}") @@ -140,7 +141,6 @@ def test_build_activity_query(mocker, monday_requester): def test_build_items_incremental_query(monday_requester): - object_name = "test_items" field_schema = { "id": {"type": "integer"}, @@ -151,20 +151,21 @@ def test_build_items_incremental_query(monday_requester): "text": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}, - "display_value": {"type": ["null", "string"]} + "display_value": {"type": ["null", "string"]}, } - } + }, } stream_slice = {"ids": [1, 2, 3]} built_query = monday_requester._build_items_incremental_query(object_name, field_schema, stream_slice) - assert built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," \ - "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + assert ( + built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," + "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + ) def test_get_request_headers(monday_requester): - headers = monday_requester.get_request_headers() assert headers == {"API-Version": "2024-01"} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-my-hours/components.py b/airbyte-integrations/connectors/source-my-hours/components.py index 4377ee24ff37..109b493ebd03 100644 --- a/airbyte-integrations/connectors/source-my-hours/components.py +++ b/airbyte-integrations/connectors/source-my-hours/components.py @@ -7,10 +7,12 @@ from typing import Any, Mapping, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://docs.airbyte.com/integrations/sources/my-hours # The Bearer token generated will expire in five days diff --git a/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py index 488bcf605f2e..0fcfbd6ff847 100755 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py @@ -14,6 +14,7 @@ import pytz from mysql.connector import Error + support_file_path_prefix = "/connector/integration_tests" catalog_write_file = support_file_path_prefix + "/temp/configured_catalog_copy.json" catalog_source_file = support_file_path_prefix + "/configured_catalog_template.json" @@ -22,12 +23,13 @@ abnormal_state_write_file = support_file_path_prefix + "/temp/abnormal_state_copy.json" abnormal_state_file = support_file_path_prefix + "/abnormal_state_template.json" -secret_config_file = '/connector/secrets/cat-config.json' -secret_active_config_file = support_file_path_prefix + '/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/cat-config-cdc.json' -secret_active_config_cdc_file = support_file_path_prefix + '/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/cat-config.json" +secret_active_config_file = support_file_path_prefix + "/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/cat-config-cdc.json" +secret_active_config_cdc_file = support_file_path_prefix + "/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') @contextmanager def connect_to_db(): @@ -36,11 +38,7 @@ def connect_to_db(): conn = None try: conn = mysql.connector.connect( - database=None, - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + database=None, user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") yield conn @@ -54,6 +52,7 @@ def connect_to_db(): conn.close() print("Database connection closed") + def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: insert_query = f"INSERT INTO {schema_name}.{table_name} (id, name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE id=id" try: @@ -66,6 +65,7 @@ def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[ print(f"Error inserting records: {error}") conn.rollback() + def create_schema(conn, schema_name: str) -> None: create_schema_query = f"CREATE DATABASE IF NOT EXISTS {schema_name}" try: @@ -77,32 +77,34 @@ def create_schema(conn, schema_name: str) -> None: print(f"Error creating database: {error}") conn.rollback() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path(support_file_path_prefix + "/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn, schema_name: str, table_name: str) -> None: create_table_query = f""" CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( @@ -119,45 +121,44 @@ def create_table(conn, schema_name: str, table_name: str) -> None: print(f"Error creating table: {error}") conn.rollback() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] - table_name = 'id_and_name_cat' + new_records = [("4", "four"), ("5", "five")] + table_name = "id_and_name_cat" with connect_to_db() as conn: insert_records(conn, schema_name, table_name, new_records) + def setup(): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] with connect_to_db() as conn: create_schema(conn, schema_name) create_table(conn, schema_name, table_name) insert_records(conn, schema_name, table_name, records) + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_schemas_with_prefix(conn, date_prefix): query = f""" SELECT schema_name @@ -177,19 +178,22 @@ def delete_schemas_with_prefix(conn, date_prefix): print(f"An error occurred in deleting schema: {e}") sys.exit(1) + def teardown() -> None: today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") with connect_to_db() as conn: delete_schemas_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: schema_name = load_schema_name_from_catalog() print(f"delete database {schema_name}") with connect_to_db() as conn: delete_schemas_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py index ea1ca1161ee2..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py @@ -5,10 +5,10 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) @pytest.fixture(scope="session", autouse=True) def connector_setup(): - yield diff --git a/airbyte-integrations/connectors/source-netsuite/main.py b/airbyte-integrations/connectors/source-netsuite/main.py index 492266da15e2..5d7d745b82af 100644 --- a/airbyte-integrations/connectors/source-netsuite/main.py +++ b/airbyte-integrations/connectors/source-netsuite/main.py @@ -4,5 +4,6 @@ from source_netsuite.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-netsuite/setup.py b/airbyte-integrations/connectors/source-netsuite/setup.py index e16d4d5b270f..682252ab2124 100644 --- a/airbyte-integrations/connectors/source-netsuite/setup.py +++ b/airbyte-integrations/connectors/source-netsuite/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", "requests-oauthlib", diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py index 610adece4944..e7f64474b9b2 100644 --- a/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py @@ -9,15 +9,15 @@ from typing import Any, List, Mapping, Tuple, Union import requests +from requests_oauthlib import OAuth1 + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from requests_oauthlib import OAuth1 from source_netsuite.constraints import CUSTOM_INCREMENTAL_CURSOR, INCREMENTAL_CURSOR, META_PATH, RECORD_PATH, SCHEMA_HEADERS from source_netsuite.streams import CustomIncrementalNetsuiteStream, IncrementalNetsuiteStream, NetsuiteStream class SourceNetsuite(AbstractSource): - logger: logging.Logger = logging.getLogger("airbyte") def auth(self, config: Mapping[str, Any]) -> OAuth1: @@ -109,7 +109,6 @@ def generate_stream( window_in_days: int, max_retry: int = 3, ) -> Union[NetsuiteStream, IncrementalNetsuiteStream, CustomIncrementalNetsuiteStream]: - input_args = { "auth": auth, "object_name": object_name, diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py index 57ab51643782..4d59ac547034 100644 --- a/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py @@ -9,8 +9,9 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union import requests -from airbyte_cdk.sources.streams.http import HttpStream from requests_oauthlib import OAuth1 + +from airbyte_cdk.sources.streams.http import HttpStream from source_netsuite.constraints import ( CUSTOM_INCREMENTAL_CURSOR, INCREMENTAL_CURSOR, @@ -159,7 +160,6 @@ def parse_response( next_page_token: Mapping[str, Any] = None, **kwargs, ) -> Iterable[Mapping]: - records = response.json().get("items") request_kwargs = self.request_kwargs(stream_slice, next_page_token) if records: diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-notion/main.py b/airbyte-integrations/connectors/source-notion/main.py index 671d6cd692fa..0c88cd7df7d3 100644 --- a/airbyte-integrations/connectors/source-notion/main.py +++ b/airbyte-integrations/connectors/source-notion/main.py @@ -4,5 +4,6 @@ from source_notion.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 5b92270c10e2..d93ce9fb56d2 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -9,13 +9,15 @@ import pendulum import pydantic import requests +from requests import HTTPError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import CheckpointMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException -from requests import HTTPError + # maximum block hierarchy recursive request depth MAX_BLOCK_DEPTH = 30 @@ -27,7 +29,6 @@ class NotionAvailabilityStrategy(HttpAvailabilityStrategy): """ def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: - reasons_for_codes: Dict[int, str] = { requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " "Please make sure your integration has read access for the resources you are trying to sync" @@ -36,7 +37,6 @@ def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, s class NotionStream(HttpStream, ABC): - url_base = "https://api.notion.com/v1/" primary_key = "id" @@ -146,7 +146,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class StateValueWrapper(pydantic.BaseModel): - stream: T state_value: str max_cursor_time: Any = "" @@ -166,7 +165,6 @@ def dict(self, **kwargs): class IncrementalNotionStream(NotionStream, CheckpointMixin, ABC): - cursor_field = "last_edited_time" http_method = "POST" diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py index 6f5965486181..be48cadc1a21 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py @@ -8,52 +8,79 @@ def test_users_stream_transformation(): input_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "user": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } output_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "info": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } assert NotionUserTransformation().transform(input_record) == output_record def test_notion_properties_transformation(): input_record = { - "id": "123", "properties": { - "Due date": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - }, + "id": "123", + "properties": { + "Due date": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, "Status": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, } output_record = { - "id": "123", "properties": [ + "id": "123", + "properties": [ { - "name": "Due date", "value": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - } + "name": "Due date", + "value": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, }, { "name": "Status", "value": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } - ] + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, + ], } assert NotionPropertiesTransformation().transform(input_record) == output_record @@ -64,55 +91,38 @@ def test_notion_properties_transformation(): {"id": "3", "last_edited_time": "2022-01-04T00:00:00.000Z"}, ] + @pytest.fixture def data_feed_config(): return NotionDataFeedFilter(parameters={}, config={"start_date": "2021-01-01T00:00:00.000Z"}) + @pytest.mark.parametrize( "state_value, expected_return", [ - ( - "2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z" - ), - ( - "2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z" - ), - ( - {}, "2021-01-01T00:00:00.000Z" - ) + ("2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z"), + ("2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z"), + ({}, "2021-01-01T00:00:00.000Z"), ], - ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"] + ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"], ) def test_data_feed_get_filter_date(data_feed_config, state_value, expected_return): start_date = data_feed_config.config["start_date"] - + result = data_feed_config._get_filter_date(start_date, state_value) assert result == expected_return, f"Expected {expected_return}, but got {result}." -@pytest.mark.parametrize("stream_state,stream_slice,expected_records", [ - ( - {"last_edited_time": "2022-01-01T00:00:00.000Z"}, - {"id": "some_id"}, - state_test_records - ), - ( - {"last_edited_time": "2022-01-03T00:00:00.000Z"}, - {"id": "some_id"}, - [state_test_records[-2], state_test_records[-1]] - ), - ( - {"last_edited_time": "2022-01-05T00:00:00.000Z"}, - {"id": "some_id"}, - [] - ), - ( - {}, - {"id": "some_id"}, - state_test_records - ) -], -ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"]) +@pytest.mark.parametrize( + "stream_state,stream_slice,expected_records", + [ + ({"last_edited_time": "2022-01-01T00:00:00.000Z"}, {"id": "some_id"}, state_test_records), + ({"last_edited_time": "2022-01-03T00:00:00.000Z"}, {"id": "some_id"}, [state_test_records[-2], state_test_records[-1]]), + ({"last_edited_time": "2022-01-05T00:00:00.000Z"}, {"id": "some_id"}, []), + ({}, {"id": "some_id"}, state_test_records), + ], + ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"], +) def test_data_feed_filter_records(data_feed_config, stream_state, stream_slice, expected_records): filtered_records = data_feed_config.filter_records(state_test_records, stream_state, stream_slice) assert filtered_records == expected_records, "Filtered records do not match the expected records." diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py index 7b323ed72257..46b97f977cb2 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py @@ -10,11 +10,12 @@ import freezegun import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException from pytest import fixture, mark from source_notion.streams import Blocks, IncrementalNotionStream, NotionStream, Pages +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException + @pytest.fixture def patch_base_class(mocker): @@ -79,11 +80,7 @@ def test_http_method(patch_base_class): @pytest.mark.parametrize( "response_json, expected_output", - [ - ({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), - ({"has_more": False}, None), - ({}, None) - ], + [({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), ({"has_more": False}, None), ({}, None)], ids=["Next_page_token exists with cursor", "No next_page_token", "No next_page_token"], ) def test_next_page_token(patch_base_class, response_json, expected_output): @@ -454,21 +451,149 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, def test_block_record_transformation(): stream = Blocks(parent=None, config=MagicMock()) response_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, - "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "page": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } expected_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], - "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "info": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } assert stream.transform(response_record) == expected_record diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py index c270f0894e5e..3a5696e77574 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_notion.source import SourceNotion +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.mark.parametrize( "config, expected_token", @@ -29,8 +30,7 @@ def test_get_authenticator(config, expected_token): def test_streams(): source = SourceNotion() - config_mock = {"start_date": "2020-01-01T00:00:00.000Z", - "credentials": {"auth_type": "token", "token": "abcd"}} + config_mock = {"start_date": "2020-01-01T00:00:00.000Z", "credentials": {"auth_type": "token", "token": "abcd"}} streams = source.streams(config_mock) expected_streams_number = 5 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py index 6f2ccc74201a..6541b9a7db2c 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-okta/main.py b/airbyte-integrations/connectors/source-okta/main.py index 7b372b3507e7..dabf89ea556f 100644 --- a/airbyte-integrations/connectors/source-okta/main.py +++ b/airbyte-integrations/connectors/source-okta/main.py @@ -4,5 +4,6 @@ from source_okta.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-okta/source_okta/components.py b/airbyte-integrations/connectors/source-okta/source_okta/components.py index bfe858878197..1f95fcaba7ad 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/components.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/components.py @@ -9,6 +9,7 @@ import jwt import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py b/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py index 82fcef527e4b..eb7e58a41d68 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index b550116424cc..ed2e1e6ebc47 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py index 0211ef7e482f..9cf815ff3dda 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py @@ -5,11 +5,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_okta.config_migration import OktaConfigMigration from source_okta.source import SourceOkta +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + CMD = "check" SOURCE: Source = SourceOkta() diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py index 546e50c59041..9b880749910f 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py @@ -9,10 +9,11 @@ import pytest import requests -from airbyte_cdk.sources.streams import Stream from source_okta.components import CustomBearerAuthenticator, CustomOauth2Authenticator from source_okta.source import SourceOkta +from airbyte_cdk.sources.streams import Stream + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceOkta() diff --git a/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-orb/components.py b/airbyte-integrations/connectors/source-orb/components.py index 3732d1f5954a..cf349c394936 100644 --- a/airbyte-integrations/connectors/source-orb/components.py +++ b/airbyte-integrations/connectors/source-orb/components.py @@ -23,7 +23,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Record: - # for each top level response record, there can be multiple sub-records depending # on granularity and other input params. This function yields one transformed record # for each subrecord in the response. @@ -63,7 +62,6 @@ class SubscriptionUsagePartitionRouter(StreamSlicer): config: Config def stream_slices(self) -> Iterable[StreamSlice]: - """ This stream is sliced per `subscription_id` and day, as well as `billable_metric_id` if a grouping key is provided. This is because the API only supports a diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/main.py b/airbyte-integrations/connectors/source-outbrain-amplify/main.py index dc58d39d5bcd..4b31ffa82fcf 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/main.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/main.py @@ -4,5 +4,6 @@ from source_outbrain_amplify.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py index 18248b027d46..4fa2955b1a75 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py @@ -5,9 +5,10 @@ from typing import Any, Mapping import requests -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from requests.auth import HTTPBasicAuth +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + class OutbrainAmplifyAuthenticator(TokenAuthenticator): def __init__(self, config, url_base): diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py index ae0a7b4d6c70..628314fc4659 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -15,6 +16,7 @@ from .auth import OutbrainAmplifyAuthenticator + DEFAULT_END_DATE = pendulum.now() DEFAULT_GEO_LOCATION_BREAKDOWN = "region" DEFAULT_REPORT_GRANULARITY = "daily" @@ -122,7 +124,6 @@ def name(self) -> str: def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -174,7 +175,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -223,7 +223,6 @@ def name(self) -> str: def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -267,7 +266,6 @@ def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -323,7 +321,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -376,7 +373,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -437,7 +433,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -500,7 +495,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -569,7 +563,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -635,7 +628,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -696,7 +688,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -759,7 +750,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -820,7 +810,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -883,7 +872,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -946,7 +934,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -1011,7 +998,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -1051,7 +1037,6 @@ def path( class IncrementalOutbrainAmplifyStream(OutbrainAmplifyStream, ABC): - state_checkpoint_interval = None @property @@ -1098,7 +1083,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # Budget for Marketers stream. # 1. Budget stream based on marketers id. - stream.extend([BudgetsForMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config))]), + (stream.extend([BudgetsForMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config))]),) # Promoted Links stream. # 1. Promoted Links stream for campaigns. diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py index 5f06ad6891b3..f545aa24ff73 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_outbrain_amplify.source import IncrementalOutbrainAmplifyStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outreach/main.py b/airbyte-integrations/connectors/source-outreach/main.py index a5b35a64025f..3fbcc84e8fb7 100644 --- a/airbyte-integrations/connectors/source-outreach/main.py +++ b/airbyte-integrations/connectors/source-outreach/main.py @@ -4,5 +4,6 @@ from source_outreach.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/components.py b/airbyte-integrations/connectors/source-outreach/source_outreach/components.py index 48ee70c5c1b6..2fe67a2a6bbd 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/components.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/components.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py index f9daaa87c1e4..a3af865c1b7f 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_outreach import SourceOutreach +from airbyte_cdk.entrypoint import launch + def run(): source = SourceOutreach() diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 3faed388f99e..4f55b84029a8 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pagerduty/main.py b/airbyte-integrations/connectors/source-pagerduty/main.py index 22537946cc8c..15c7eed00df9 100644 --- a/airbyte-integrations/connectors/source-pagerduty/main.py +++ b/airbyte-integrations/connectors/source-pagerduty/main.py @@ -4,5 +4,6 @@ from source_pagerduty.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py index f73afe9c6592..21f3c24f9f92 100644 --- a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_pagerduty import SourcePagerduty +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePagerduty() diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py index 96ff2d87bbfb..2008ace84e36 100644 --- a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-partnerstack/main.py b/airbyte-integrations/connectors/source-partnerstack/main.py index d22642a3ee66..6cc90fad696d 100644 --- a/airbyte-integrations/connectors/source-partnerstack/main.py +++ b/airbyte-integrations/connectors/source-partnerstack/main.py @@ -4,5 +4,6 @@ from source_partnerstack.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py index 1b7dad130b55..9fc7e58ecb5f 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_partnerstack import SourcePartnerstack +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePartnerstack() diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py index 79d39b4f690b..baacf122c6e7 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py index c371024b7ff9..02aa30582dd4 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py @@ -62,7 +62,6 @@ def read_json(filepath): def main(): - operation = sys.argv[1] CREDS = read_json("../secrets/config.json") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py index cf9d8d86e92d..2228f436d5f5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py @@ -8,6 +8,7 @@ # %% import requests + logging.basicConfig(level=logging.DEBUG) # %% diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py index 6a2f46c3b524..8be90ea420c7 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py @@ -82,7 +82,6 @@ def read_json(filepath): def main(): - CREDS = read_json("../secrets/config.json") client_id = CREDS.get("client_id") secret_id = CREDS.get("client_secret") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py index d8067ec1e2bf..1568d1ace9b3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -34,6 +34,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait + # from pprint import pprint @@ -104,7 +105,6 @@ def read_json(filepath): def get_api_token(): - client_id = CREDS.get("client_id") secret = CREDS.get("client_secret") @@ -126,7 +126,6 @@ def random_digits(digits): def make_payment(): - # generate new invoice_number PAYMENT_DATA["transactions"][0]["invoice_number"] = random_digits(11) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 06823a4a71e5..302035ec0bc2 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -4,5 +4,6 @@ from source_paypal_transaction.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py index af883e9c1c19..814bfd5b0c43 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py @@ -10,11 +10,13 @@ import backoff import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py index 1a6d4cc56c0e..a6433dd29bdf 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_paypal_transaction import SourcePaypalTransaction +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePaypalTransaction() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 07d75f4bc281..c71a7fb2a782 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. @@ -13,6 +14,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourcePaypalTransaction(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py index dd19b6306e77..1bf0f5692aaa 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py @@ -7,58 +7,59 @@ import pytest import requests import requests_mock -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from source_paypal_transaction.components import PayPalOauth2Authenticator +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + @pytest.fixture def mock_authenticator(): return PayPalOauth2Authenticator( config={}, parameters={}, - client_id='test_client_id', - client_secret='test_client_secret', - token_refresh_endpoint='https://test.token.endpoint', - grant_type='test_grant_type' + client_id="test_client_id", + client_secret="test_client_secret", + token_refresh_endpoint="https://test.token.endpoint", + grant_type="test_grant_type", ) + def test_get_refresh_access_token_response(mock_authenticator): - expected_response_json = {'access_token': 'test_access_token', 'expires_in': 3600} + expected_response_json = {"access_token": "test_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=expected_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=expected_response_json, status_code=200) # Call _get_refresh method mock_authenticator._get_refresh_access_token_response() - - assert mock_authenticator.access_token == expected_response_json['access_token'] + + assert mock_authenticator.access_token == expected_response_json["access_token"] + def test_token_expiration(mock_authenticator): # Mock response for initial token request - initial_response_json = {'access_token': 'initial_access_token', 'expires_in': 1} + initial_response_json = {"access_token": "initial_access_token", "expires_in": 1} # Mock response for token refresh request - refresh_response_json = {'access_token': 'refreshed_access_token', 'expires_in': 3600} + refresh_response_json = {"access_token": "refreshed_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - - mock_request.post('https://test.token.endpoint', json=initial_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=initial_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the initial access token is set correctly - assert mock_authenticator.access_token == initial_response_json['access_token'] + assert mock_authenticator.access_token == initial_response_json["access_token"] time.sleep(2) - mock_request.post('https://test.token.endpoint', json=refresh_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=refresh_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the access token is refreshed - assert mock_authenticator.access_token == refresh_response_json['access_token'] + assert mock_authenticator.access_token == refresh_response_json["access_token"] def test_backoff_retry(mock_authenticator, caplog): - - mock_response = {'access_token': 'test_access_token', 'expires_in': 3600} + mock_response = {"access_token": "test_access_token", "expires_in": 3600} mock_reason = "Too Many Requests" - + with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=mock_response, status_code=429, reason=mock_reason) + mock_request.post("https://test.token.endpoint", json=mock_response, status_code=429, reason=mock_reason) with caplog.at_level(logging.INFO): try: mock_authenticator._get_refresh_access_token_response() @@ -67,6 +68,7 @@ def test_backoff_retry(mock_authenticator, caplog): else: pytest.fail("Expected DefaultBackoffException to be raised") + @pytest.fixture def authenticator_parameters(): return { @@ -75,14 +77,12 @@ def authenticator_parameters(): "config": {}, "parameters": {}, "token_refresh_endpoint": "https://test.token.endpoint", - "grant_type": "test_grant_type" + "grant_type": "test_grant_type", } + def test_get_headers(authenticator_parameters): expected_basic_auth = "Basic dGVzdF9jbGllbnRfaWQ6dGVzdF9jbGllbnRfc2VjcmV0" authenticator = PayPalOauth2Authenticator(**authenticator_parameters) headers = authenticator.get_headers() assert headers == {"Authorization": expected_basic_auth} - - - diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index 06dd08dc74a6..bd2f9c89836a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -11,21 +11,21 @@ @pytest.fixture(name="config") def config_fixture(): - #From File test + # From File test # with open('../secrets/config.json') as f: # return json.load(f) - #Mock test + # Mock test return { - "client_id": "your_client_id", - "client_secret": "your_client_secret", - "start_date": "2024-01-30T00:00:00Z", - "end_date": "2024-02-01T00:00:00Z", - "dispute_start_date": "2024-02-01T00:00:00.000Z", - "dispute_end_date": "2024-02-05T23:59:00.000Z", - "buyer_username": "Your Buyer email", - "buyer_password": "Your Buyer Password", - "payer_id": "ypur ACCOUNT ID", - "is_sandbox": True + "client_id": "your_client_id", + "client_secret": "your_client_secret", + "start_date": "2024-01-30T00:00:00Z", + "end_date": "2024-02-01T00:00:00Z", + "dispute_start_date": "2024-02-01T00:00:00.000Z", + "dispute_end_date": "2024-02-05T23:59:00.000Z", + "buyer_username": "Your Buyer email", + "buyer_password": "Your Buyer Password", + "payer_id": "ypur ACCOUNT ID", + "is_sandbox": True, } @@ -33,21 +33,24 @@ def config_fixture(): def source_fixture(): return SourcePaypalTransaction() + def validate_date_format(date_str, format): try: datetime.strptime(date_str, format) return True except ValueError: return False - + + def test_date_formats_in_config(config): start_date_format = "%Y-%m-%dT%H:%M:%SZ" dispute_date_format = "%Y-%m-%dT%H:%M:%S.%fZ" - assert validate_date_format(config['start_date'], start_date_format), "Start date format is incorrect" - assert validate_date_format(config['end_date'], start_date_format), "End date format is incorrect" - assert validate_date_format(config['dispute_start_date'], dispute_date_format), "Dispute start date format is incorrect" - assert validate_date_format(config['dispute_end_date'], dispute_date_format), "Dispute end date format is incorrect" + assert validate_date_format(config["start_date"], start_date_format), "Start date format is incorrect" + assert validate_date_format(config["end_date"], start_date_format), "End date format is incorrect" + assert validate_date_format(config["dispute_start_date"], dispute_date_format), "Dispute start date format is incorrect" + assert validate_date_format(config["dispute_end_date"], dispute_date_format), "Dispute end date format is incorrect" + @pytest.fixture(name="logger_mock") def logger_mock_fixture(): - return patch("source_paypal_transactions.source.AirbyteLogger") \ No newline at end of file + return patch("source_paypal_transactions.source.AirbyteLogger") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py index 958db41262da..ddf832641c2d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py @@ -8,6 +8,7 @@ import pytest import requests import requests_mock + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean @@ -27,23 +28,23 @@ class CursorPaginationStrategy(PaginationStrategy): stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating decoder (Decoder): decoder to decode the response """ + cursor_value: Union[InterpolatedString, str] config: Config parameters: Mapping[str, Any] page_size: Optional[int] = None stop_condition: Optional[Union[InterpolatedBoolean, str]] = None decoder: Decoder = field(default_factory=JsonDecoder) - + def __post_init__(self): if isinstance(self.cursor_value, str): self.cursor_value = InterpolatedString.create(self.cursor_value, parameters=self.parameters) if isinstance(self.stop_condition, str): self.stop_condition = InterpolatedBoolean(condition=self.stop_condition, parameters=self.parameters) - + @property def initial_token(self) -> Optional[Any]: return None - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) @@ -51,53 +52,45 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin headers["link"] = response.links print("STOP CONDITION", self.stop_condition) - + if self.stop_condition: should_stop = self.stop_condition.eval(self.config, response=decoded_response, headers=headers, last_records=last_records) if should_stop: print("Stopping...") return None - + # Update cursor_value with the next_id from the response self.cursor_value = InterpolatedString.create(decoded_response.get("next_id"), parameters=self.parameters) token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response, headers=headers) print("TOKEN", token) return token if token else None - + def reset(self): pass - + def get_page_size(self) -> Optional[int]: return self.page_size @pytest.fixture def mock_responses(): - return [ - "token_page_init.json", - "token_PAY-0L38757939422510JMW5ZJVA.json", - "token_PAYID-MW5XXZY5YL87592N34454913.json" - ] + return ["token_page_init.json", "token_PAY-0L38757939422510JMW5ZJVA.json", "token_PAYID-MW5XXZY5YL87592N34454913.json"] + @pytest.fixture -def cursor_pagination_strategy(mock_responses, stop_condition = None): +def cursor_pagination_strategy(mock_responses, stop_condition=None): parameters = {} decoder = JsonDecoder(parameters=parameters) cursor_value = "start_id" # Initialize with a default value - + for response_file in mock_responses: if cursor_value == "start_id": cursor_value = load_mock_data(response_file).get("next_id") else: break # Stop after getting the next_id from the first response - + return CursorPaginationStrategy( - cursor_value=cursor_value, - config={}, - parameters=parameters, - page_size=3, - stop_condition=stop_condition, - decoder=decoder + cursor_value=cursor_value, config={}, parameters=parameters, page_size=3, stop_condition=stop_condition, decoder=decoder ) @@ -105,6 +98,7 @@ def load_mock_data(filename): with open(os.path.join("./unit_tests/test_files", filename), "r") as file: return json.load(file) + def test_cursor_pagination(cursor_pagination_strategy, mock_responses): with requests_mock.Mocker() as m: base_url = "http://example.com/api/resource" @@ -126,21 +120,21 @@ def test_cursor_pagination(cursor_pagination_strategy, mock_responses): if i < len(mock_responses) - 1: next_id = load_mock_data(response_file)["next_id"] print("FOUND NEXT ID:", next_id) - + else: next_id = None - cursor_pagination_strategy(mock_responses, stop_condition = True) + cursor_pagination_strategy(mock_responses, stop_condition=True) # Make API call and process response response = requests.get(url) print("GET RESPONSE:", response) assert response.status_code == 200 - + decoded_response = response.json() last_records = decoded_response["payments"] next_id = cursor_pagination_strategy.next_page_token(response, last_records) print("NEXT ID:", next_id) - + # Verify the pagination stopped assert next_id is None print("No more pages") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py index 05b98d04f90a..1166076c7972 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py @@ -6,6 +6,7 @@ import pytest import requests import requests_mock + from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy @@ -29,24 +30,25 @@ def reset(self): def get_page_size(self): return self.page_size + @pytest.fixture def mock_pagination_strategy(): return MockPaginationStrategy(page_size=500) + @pytest.fixture def paginator(): pagination_strategy = MockPaginationStrategy(page_size=3) return DefaultPaginator( - pagination_strategy=pagination_strategy, - config={}, - url_base="http://example.com/v1/reporting/transactions", - parameters={} + pagination_strategy=pagination_strategy, config={}, url_base="http://example.com/v1/reporting/transactions", parameters={} ) - + + def load_mock_data(page): with open(f"./unit_tests/test_files/page_{page}.json", "r") as file: return file.read() + # Test to verify pagination logic transitions from page 1 to page 2 def test_pagination_logic(paginator): page_1_data = load_mock_data(1) @@ -54,7 +56,7 @@ def test_pagination_logic(paginator): paginator_url_1 = f"{paginator.url_base.string}?page=1&page_size={paginator.pagination_strategy.get_page_size}" paginator_url_2 = f"{paginator.url_base.string}?page=2&page_size={paginator.pagination_strategy.get_page_size}" - + with requests_mock.Mocker() as m: m.get(paginator_url_1, text=page_1_data, status_code=200) m.get(paginator_url_2, text=page_2_data, status_code=200) @@ -64,16 +66,14 @@ def test_pagination_logic(paginator): response_page_2 = requests.get(paginator_url_2) response_page_2._content = str.encode(page_2_data) - # Simulate getting the next page token from page 1's response next_page_token_page_1 = paginator.next_page_token(response_page_1, []) print("NEXT PAGE TOKEN", next_page_token_page_1) # Assert that the next page token indicates moving to page 2 - assert next_page_token_page_1['next_page_token'] == 2, "Failed to transition from page 1 to page 2" + assert next_page_token_page_1["next_page_token"] == 2, "Failed to transition from page 1 to page 2" - # Check that the correct page size is used in requests and that we have the right number of pages - assert len(m.request_history) == 2 - assert "page_size=3" in m.request_history[0].url - assert "page_size=3" in m.request_history[1].url \ No newline at end of file + assert len(m.request_history) == 2 + assert "page_size=3" in m.request_history[0].url + assert "page_size=3" in m.request_history[1].url diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pinterest/main.py b/airbyte-integrations/connectors/source-pinterest/main.py index aff013c70319..59806e3eb585 100644 --- a/airbyte-integrations/connectors/source-pinterest/main.py +++ b/airbyte-integrations/connectors/source-pinterest/main.py @@ -4,5 +4,6 @@ from source_pinterest.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py index f31a115c09a9..91a946b0b711 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py @@ -9,12 +9,14 @@ import backoff import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py index d689a4643205..bfdcd32d582b 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py @@ -6,6 +6,7 @@ import pendulum import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py index 892b5fc652f2..f469def56e9c 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -8,9 +8,10 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional from urllib.parse import urljoin -import airbyte_cdk.sources.utils.casing as casing import backoff import requests + +import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index bdcb8f7c00ce..44de3103bdc7 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping import pendulum + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream @@ -32,6 +33,7 @@ ) from .streams import PinterestStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py index d90f3e0862e6..94045a7e7013 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk import AirbyteTracedException, BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import WaitTimeFromHeaderBackoffStrategy @@ -20,6 +21,7 @@ from .utils import get_analytics_columns, to_datetime_str + # For Pinterest analytics streams rate limit is 300 calls per day / per user. # once hit - response would contain `code` property with int. MAX_RATE_LIMIT_CODE = 8 diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py index 0ca0151e2db8..271de750bcff 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py @@ -5,11 +5,12 @@ from typing import Any, Mapping from unittest.mock import MagicMock -from airbyte_cdk.sources.streams import Stream from pytest import fixture from source_pinterest.reports import CampaignAnalyticsReport from source_pinterest.source import SourcePinterest +from airbyte_cdk.sources.streams import Stream + @fixture def test_config() -> Mapping[str, str]: diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py index 86cc6f23f8f4..6f2f2207cdb8 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py @@ -7,16 +7,19 @@ import pendulum import pytest import requests +from requests import Response +from source_pinterest.python_stream_auth import PinterestOauthAuthenticator + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from requests import Response -from source_pinterest.python_stream_auth import PinterestOauthAuthenticator + LOGGER = logging.getLogger(__name__) resp = Response() + class TestPinterestOauthAuthenticator: """ Test class for custom PinterestOauthAuthenticator, derived from the CDK's Oauth2Authenticator class. @@ -69,7 +72,9 @@ def test_refresh_access_token_invalid_or_expired(self, mocker, oauth): mocker.patch.object(resp, "status_code", 400) mocker.patch.object(oauth, "_wrap_refresh_token_exception", return_value=True) - with pytest.raises(AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings."): + with pytest.raises( + AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." + ): oauth.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py index 3dd38604a86e..bfa60d227e31 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py @@ -7,11 +7,12 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from pytest import fixture from source_pinterest.streams import IncrementalPinterestSubStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + from .conftest import get_stream_by_name from .utils import create_requests_response @@ -135,7 +136,8 @@ def test_semi_incremental_read(requests_mock, test_config, start_date, stream_st stream.state = stream_state actual_records = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) for record in stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice) ] assert actual_records == expected_records diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index df5c903ee347..16df2e7ad9cf 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -24,7 +24,8 @@ ) from source_pinterest.utils import get_analytics_columns -os.environ["REQUEST_CACHE_PATH"] = '/tmp' + +os.environ["REQUEST_CACHE_PATH"] = "/tmp" def test_request_body_json(analytics_report_stream, date_range): @@ -79,18 +80,20 @@ def test_streams(test_config): def test_custom_streams(test_config): config = copy.deepcopy(test_config) - config['custom_reports'] = [{ - "name": "vadim_report", - "level": "AD_GROUP", - "granularity": "MONTH", - "click_window_days": 30, - "engagement_window_days": 30, - "view_window_days": 30, - "conversion_report_time": "TIME_OF_CONVERSION", - "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], - "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], - "start_date": "2023-01-08" - }] + config["custom_reports"] = [ + { + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08", + } + ] source = SourcePinterest() streams = source.streams(config) expected_streams_number = 33 @@ -100,18 +103,18 @@ def test_custom_streams(test_config): @pytest.mark.parametrize( ("report_name", "expected_level"), ( - [CampaignAnalyticsReport, 'CAMPAIGN'], - [CampaignTargetingReport, 'CAMPAIGN_TARGETING'], - [AdvertiserReport, 'ADVERTISER'], - [AdvertiserTargetingReport, 'ADVERTISER_TARGETING'], - [AdGroupReport, 'AD_GROUP'], - [AdGroupTargetingReport, 'AD_GROUP_TARGETING'], - [PinPromotionReport, 'PIN_PROMOTION'], - [PinPromotionTargetingReport, 'PIN_PROMOTION_TARGETING'], - [ProductGroupReport, 'PRODUCT_GROUP'], - [ProductGroupTargetingReport, 'PRODUCT_GROUP_TARGETING'], - [ProductItemReport, 'PRODUCT_ITEM'], - [KeywordReport, 'KEYWORD'] + [CampaignAnalyticsReport, "CAMPAIGN"], + [CampaignTargetingReport, "CAMPAIGN_TARGETING"], + [AdvertiserReport, "ADVERTISER"], + [AdvertiserTargetingReport, "ADVERTISER_TARGETING"], + [AdGroupReport, "AD_GROUP"], + [AdGroupTargetingReport, "AD_GROUP_TARGETING"], + [PinPromotionReport, "PIN_PROMOTION"], + [PinPromotionTargetingReport, "PIN_PROMOTION_TARGETING"], + [ProductGroupReport, "PRODUCT_GROUP"], + [ProductGroupTargetingReport, "PRODUCT_GROUP_TARGETING"], + [ProductItemReport, "PRODUCT_ITEM"], + [KeywordReport, "KEYWORD"], ), ) def test_level(test_config, report_name, expected_level): diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 6012f9d1b211..5e615e7c2e8d 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -5,9 +5,10 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_pinterest.source import SourcePinterest +from airbyte_cdk.utils import AirbyteTracedException + def test_check_connection(requests_mock, test_config): requests_mock.get("https://api.pinterest.com/v5/boards", status_code=200) @@ -30,8 +31,7 @@ def test_check_connection_expired_token(requests_mock, test_config): logger_mock = MagicMock() assert source.check_connection(logger_mock, test_config) == ( False, - "Unable to connect to stream boards - 401 Client Error: None " - "for url: https://api.pinterest.com/v5/oauth/token", + "Unable to connect to stream boards - 401 Client Error: None " "for url: https://api.pinterest.com/v5/oauth/token", ) diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 80f5d4aa37a7..b1ecc6bfef44 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -8,10 +8,6 @@ import pytest import requests -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models.airbyte_protocol import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from source_pinterest.streams import ( AnalyticsApiBackoffStrategyDecorator, NonJSONResponse, @@ -22,9 +18,15 @@ ) from source_pinterest.utils import get_analytics_columns +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models.airbyte_protocol import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + from .conftest import get_stream_by_name from .utils import create_requests_response + os.environ["REQUEST_CACHE_PATH"] = "/tmp" _ANY_STREAM_NAME = "any_stream_name" _RETRY_AFTER_HEADER = "XRetry-After" @@ -80,7 +82,8 @@ def test_parse_response_with_sensitive_data(requests_mock, test_config): json={"items": [{"id": "CatalogsFeeds1", "credentials": {"password": "bla"}}]}, ) actual_response = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) ] assert actual_response == [{"id": "CatalogsFeeds1"}] @@ -122,7 +125,9 @@ def test_response_action(requests_mock, patch_base_class, http_status, expected_ ), ) @patch("time.sleep", return_value=None) -def test_declarative_stream_response_action_on_max_rate_limit_error(mock_sleep, requests_mock, test_response, status_code, expected_response_action): +def test_declarative_stream_response_action_on_max_rate_limit_error( + mock_sleep, requests_mock, test_response, status_code, expected_response_action +): response_mock = create_requests_response(requests_mock, status_code, {}) error_handler = PinterestErrorHandler(logger=MagicMock(), stream_name="any_stream_name") assert error_handler.interpret_response(response_mock).response_action == expected_response_action diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pipedrive/main.py b/airbyte-integrations/connectors/source-pipedrive/main.py index 64fe456c34fd..4b1f33f52ccc 100644 --- a/airbyte-integrations/connectors/source-pipedrive/main.py +++ b/airbyte-integrations/connectors/source-pipedrive/main.py @@ -4,5 +4,6 @@ from source_pipedrive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py index 961438884da6..2f34fa080e10 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping, Union import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py index 68d288dbaeef..fd9c66cb3824 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py @@ -7,11 +7,12 @@ from datetime import datetime from typing import List +from orjson import orjson +from source_pipedrive import SourcePipedrive + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson -from source_pipedrive import SourcePipedrive def _get_source(args: List[str]): diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py index 8070fbcfcc40..b109d02ac76c 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py index 3a0f562732fb..6e0d32803f45 100644 --- a/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pocket/components.py b/airbyte-integrations/connectors/source-pocket/components.py index 0d5ef1cec552..19414f6f4561 100644 --- a/airbyte-integrations/connectors/source-pocket/components.py +++ b/airbyte-integrations/connectors/source-pocket/components.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py index 56afb0875706..93a63daefa3c 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py @@ -13,6 +13,7 @@ import pytz from psycopg2 import extensions, sql + catalog_write_file = "/connector/integration_tests/temp/configured_catalog_copy.json" catalog_source_file = "/connector/integration_tests/configured_catalog_template.json" catalog_incremental_write_file = "/connector/integration_tests/temp/incremental_configured_catalog_copy.json" @@ -20,12 +21,13 @@ abnormal_state_write_file = "/connector/integration_tests/temp/abnormal_state_copy.json" abnormal_state_file = "/connector/integration_tests/abnormal_state_template.json" -secret_config_file = '/connector/secrets/config.json' -secret_active_config_file = '/connector/integration_tests/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/config_cdc.json' -secret_active_config_cdc_file = '/connector/integration_tests/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/config.json" +secret_active_config_file = "/connector/integration_tests/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/config_cdc.json" +secret_active_config_cdc_file = "/connector/integration_tests/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') def connect_to_db() -> extensions.connection: with open(secret_config_file) as f: @@ -33,11 +35,7 @@ def connect_to_db() -> extensions.connection: try: conn: extensions.connection = psycopg2.connect( - dbname=secret["database"], - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + dbname=secret["database"], user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") return conn @@ -45,6 +43,7 @@ def connect_to_db() -> extensions.connection: print(f"Error connecting to the database: {error}") sys.exit(1) + def insert_records(conn: extensions.connection, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: try: cursor = conn.cursor() @@ -64,6 +63,7 @@ def insert_records(conn: extensions.connection, schema_name: str, table_name: st finally: cursor.close() + def create_schema(conn: extensions.connection, schema_name: str) -> None: try: cursor = conn.cursor() @@ -77,24 +77,25 @@ def create_schema(conn: extensions.connection, schema_name: str) -> None: finally: cursor.close() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path("/connector/integration_tests/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["schemas"] = [schema_name] - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: @@ -104,9 +105,10 @@ def write_supporting_file(schema_name: str) -> None: secret["replication_method"]["publication"] = schema_name secret["ssl_mode"] = {} secret["ssl_mode"]["mode"] = "require" - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn: extensions.connection, schema_name: str, table_name: str) -> None: try: cursor = conn.cursor() @@ -126,40 +128,37 @@ def create_table(conn: extensions.connection, schema_name: str, table_name: str) finally: cursor.close() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] + new_records = [("4", "four"), ("5", "five")] connection = connect_to_db() - table_name = 'id_and_name_cat' + table_name = "id_and_name_cat" if connection: insert_records(connection, schema_name, table_name, new_records) connection.close() + def setup(with_cdc=False): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" # Define the records to be inserted - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] # Connect to the database connection = connect_to_db() @@ -167,7 +166,7 @@ def setup(with_cdc=False): # Create the schema create_schema(connection, schema_name) create_table(connection, schema_name, table_name) - if (with_cdc): + if with_cdc: setup_cdc(connection, replication_slot_and_publication_name=schema_name) # Insert the records insert_records(connection, schema_name, table_name, records) @@ -175,6 +174,7 @@ def setup(with_cdc=False): # Close the connection connection.close() + def replication_slot_existed(connection, replication_slot_name): cursor = connection.cursor() cursor.execute("SELECT slot_name FROM pg_replication_slots;") @@ -185,22 +185,31 @@ def replication_slot_existed(connection, replication_slot_name): return True return False + def setup_cdc(connection, replication_slot_and_publication_name): cursor = connection.cursor() if replication_slot_existed(connection, replication_slot_and_publication_name): return - create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format(sql.Literal(replication_slot_and_publication_name)) + create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format( + sql.Literal(replication_slot_and_publication_name) + ) cursor.execute(create_logical_replication_query) - alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format(sql.Identifier(replication_slot_and_publication_name)) + alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format( + sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(alter_table_replica_query) - create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format(sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name)) + create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format( + sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(create_publication_query) connection.commit() + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_cdc_with_prefix(conn, prefix): try: # Connect to the PostgreSQL database @@ -224,6 +233,7 @@ def delete_cdc_with_prefix(conn, prefix): if cursor: cursor.close() + def delete_schemas_with_prefix(conn, date_prefix): try: # Connect to the PostgreSQL database @@ -252,14 +262,16 @@ def delete_schemas_with_prefix(conn, date_prefix): finally: cursor.close() + def teardown() -> None: conn = connect_to_db() today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") delete_schemas_with_prefix(conn, formatted_yesterday) delete_cdc_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: conn = connect_to_db() schema_name = load_schema_name_from_catalog() @@ -267,6 +279,7 @@ def final_teardown() -> None: delete_schemas_with_prefix(conn, schema_name) delete_cdc_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-posthog/main.py b/airbyte-integrations/connectors/source-posthog/main.py index f7e69357d999..e6ff813e7420 100644 --- a/airbyte-integrations/connectors/source-posthog/main.py +++ b/airbyte-integrations/connectors/source-posthog/main.py @@ -4,5 +4,6 @@ from source_posthog.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py index 268ce3822d51..a7ded5e7a793 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py index a7c40deb21fe..7e1bb5bd1d0b 100644 --- a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py @@ -3,11 +3,13 @@ # import pytest as pytest +from source_posthog.components import EventsCartesianProductStreamSlicer + from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption -from source_posthog.components import EventsCartesianProductStreamSlicer + stream_slicers = [ ListPartitionRouter(values=[2331], cursor_field="project_id", config={}, parameters={}), diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py index 04bbfbafe845..5016adc4b09f 100644 --- a/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py @@ -3,9 +3,10 @@ # -from airbyte_cdk.logger import AirbyteLogger from source_posthog import SourcePosthog +from airbyte_cdk.logger import AirbyteLogger + def test_client_wrong_credentials(): source = SourcePosthog() diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py index 51f9b12de27d..e023a7a92b30 100644 --- a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-prestashop/main.py b/airbyte-integrations/connectors/source-prestashop/main.py index e51c47a996aa..1daf7c26e83b 100644 --- a/airbyte-integrations/connectors/source-prestashop/main.py +++ b/airbyte-integrations/connectors/source-prestashop/main.py @@ -4,5 +4,6 @@ from source_prestashop.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py index 0175750fc814..a3f42725cfce 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py @@ -6,10 +6,11 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from pendulum.parsing.exceptions import ParserError + from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState -from pendulum.parsing.exceptions import ParserError @dataclass diff --git a/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-public-apis/main.py b/airbyte-integrations/connectors/source-public-apis/main.py index c9796a4aa4ef..d39349fbd421 100644 --- a/airbyte-integrations/connectors/source-public-apis/main.py +++ b/airbyte-integrations/connectors/source-public-apis/main.py @@ -4,5 +4,6 @@ from source_public_apis.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py index d659c4dfe5b1..f5b17ab6ba99 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py @@ -5,10 +5,10 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor class CustomExtractor(RecordExtractor): def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: - return [{"name": cat} for cat in response.json()["categories"]] diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py index b4927fb5a5e2..951a517434cf 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_public_apis import SourcePublicApis +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePublicApis() diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py index b9925483338d..ab25aeb7a367 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/main.py b/airbyte-integrations/connectors/source-python-http-tutorial/main.py index 57dce4e0679c..e38c9bbaf28e 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/main.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/main.py @@ -4,5 +4,6 @@ from source_python_http_tutorial.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py index 35164f2108aa..9c354e0ab0ff 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py @@ -4,6 +4,7 @@ from setuptools import find_packages, setup + setup( entry_points={ "console_scripts": [ diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py index 07921116b35c..f287a682d498 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-qualaroo/components.py b/airbyte-integrations/connectors/source-qualaroo/components.py index 5e4e619d4d44..5c700af307b2 100644 --- a/airbyte-integrations/connectors/source-qualaroo/components.py +++ b/airbyte-integrations/connectors/source-qualaroo/components.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -15,7 +16,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self): - key = str(self._username.eval(self.config)).encode("latin1") token = self._password.eval(self.config).encode("latin1") encoded_credentials = b64encode(b":".join((key, token))).strip() @@ -25,7 +25,6 @@ def token(self): class CustomExtractor(RecordExtractor): def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: - extracted = [] for record in response.json(): if "answered_questions" in record: diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-quickbooks/main.py b/airbyte-integrations/connectors/source-quickbooks/main.py index abeed13585f5..bce6717c1dc9 100644 --- a/airbyte-integrations/connectors/source-quickbooks/main.py +++ b/airbyte-integrations/connectors/source-quickbooks/main.py @@ -4,5 +4,6 @@ from source_quickbooks.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py index 644d8d5b227e..77fa08020777 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-railz/main.py b/airbyte-integrations/connectors/source-railz/main.py index bfa6b9fadb2f..ec454b464ed9 100644 --- a/airbyte-integrations/connectors/source-railz/main.py +++ b/airbyte-integrations/connectors/source-railz/main.py @@ -4,5 +4,6 @@ from source_railz.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-railz/source_railz/components.py b/airbyte-integrations/connectors/source-railz/source_railz/components.py index 36a9351f859d..c7c4ef88a467 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/components.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/components.py @@ -8,6 +8,8 @@ from typing import Any, Iterable, Mapping, Optional, Union import requests +from isodate import Duration, parse_duration + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator @@ -15,7 +17,6 @@ from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator -from isodate import Duration, parse_duration @dataclass diff --git a/airbyte-integrations/connectors/source-railz/source_railz/run.py b/airbyte-integrations/connectors/source-railz/source_railz/run.py index 831b665a05da..92d5008017b4 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/run.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_railz import SourceRailz +from airbyte_cdk.entrypoint import launch + def run(): source = SourceRailz() diff --git a/airbyte-integrations/connectors/source-railz/source_railz/source.py b/airbyte-integrations/connectors/source-railz/source_railz/source.py index d5c9010ae224..642ebb4eb236 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/source.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recharge/main.py b/airbyte-integrations/connectors/source-recharge/main.py index d8ccf40b711e..5f5e6b67eed1 100644 --- a/airbyte-integrations/connectors/source-recharge/main.py +++ b/airbyte-integrations/connectors/source-recharge/main.py @@ -4,5 +4,6 @@ from source_recharge.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py index 80d19ac2eca2..d89d6281908c 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py @@ -5,9 +5,10 @@ import logging from typing import Optional, Union +from requests import Response + from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction -from requests import Response class RechargeErrorHandler(HttpStatusErrorHandler): @@ -16,7 +17,6 @@ def __init__(self, logger: logging.Logger) -> None: super().__init__(logger=logger) def interpret_response(self, response_or_exception: Optional[Union[Response, Exception]] = None) -> ErrorResolution: - if isinstance(response_or_exception, Response): content_length = int(response_or_exception.headers.get("Content-Length", 0)) incomplete_data_response = response_or_exception.status_code == 200 and content_length > len(response_or_exception.content) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index be0b9d43509d..e2aa51028f59 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -9,6 +9,7 @@ from airbyte_cdk.sources.streams import Stream from source_recharge.streams import Orders, RechargeTokenAuthenticator + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py index 39b44c35cf3d..be01b454fa1c 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py index 3581ace5712d..a014169b9c07 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + START_DATE = "2023-01-01T00:00:00Z" ACCESS_TOKEN = "test_access_token" DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py index 4522eec9675e..cb3c13212c37 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py @@ -8,6 +8,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "New_Next_Page_Token" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index d6a06768d52f..cebf1c676549 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -47,7 +47,7 @@ def with_access_token(self, access_token: str) -> RequestBuilder: def with_old_api_version(self, api_version: str) -> RequestBuilder: self._headers["X-Recharge-Version"] = api_version return self - + def with_created_min(self, value: str) -> RequestBuilder: self._query_params["created_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) return self diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py index 91ebbbcb26e6..b6055873bc81 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py @@ -5,12 +5,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, read_full_refresh + _STREAM_NAME = "collections" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 906fb220c3fc..430a27d2a0ce 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "discounts" _CURSOR_FIELD = "updated_at" @@ -31,7 +33,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py index c33a76e800c5..4093fba40a02 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "events" _CURSOR_FIELD = "created_at" @@ -32,7 +34,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index 713678584e21..64255813f503 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "onetimes" _CURSOR_FIELD = "updated_at" @@ -31,7 +33,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py index 3970bf63fad1..0991a86f1d9b 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from ..config import NOW from ..utils import StreamTestCase, read_full_refresh + _STREAM_NAME = "shop" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py index 3da0b1d06775..475f8d95ee1d 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -8,10 +8,11 @@ from typing import Any, List, Mapping, Optional from unittest import TestCase +from source_recharge import SourceRecharge + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode -from source_recharge import SourceRecharge from .config import ConfigBuilder from .request_builder import RequestBuilder diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index af3ba6d425b8..75f5536adf84 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -8,9 +8,10 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from source_recharge.source import Orders, RechargeTokenAuthenticator, SourceRecharge +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + def use_orders_deprecated_api_config( config: Mapping[str, Any] = None, @@ -88,6 +89,7 @@ def test_should_retry(self, config, http_status, headers, expected_action) -> No error_resolution = stream.get_error_handler().interpret_response(response) error_resolution.response_action == expected_action + class TestFullRefreshStreams: def generate_records(self, stream_name, count) -> Union[Mapping[str, List[Mapping[str, Any]]], Mapping[str, Any]]: if not stream_name: diff --git a/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recurly/components.py b/airbyte-integrations/connectors/source-recurly/components.py index 335c318c7944..6cebf58f980d 100644 --- a/airbyte-integrations/connectors/source-recurly/components.py +++ b/airbyte-integrations/connectors/source-recurly/components.py @@ -3,6 +3,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py index efc25f08ce82..78b220cebb18 100644 --- a/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rki-covid/main.py b/airbyte-integrations/connectors/source-rki-covid/main.py index a104106001cb..1e4d286df619 100644 --- a/airbyte-integrations/connectors/source-rki-covid/main.py +++ b/airbyte-integrations/connectors/source-rki-covid/main.py @@ -4,5 +4,6 @@ from source_rki_covid.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 65fe10330a2e..ee3cc9e34ea5 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -15,7 +16,6 @@ # Basic full refresh stream class RkiCovidStream(HttpStream, ABC): - url_base = "https://api.corona-zahlen.org/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -106,7 +106,6 @@ def path( # Basic incremental stream class IncrementalRkiCovidStream(RkiCovidStream, ABC): - state_checkpoint_interval = None @property diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py index e701d03455d4..a4058a6f3deb 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_rki_covid.source import IncrementalRkiCovidStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rss/main.py b/airbyte-integrations/connectors/source-rss/main.py index 9e21c3c97793..c8e2b7f3042c 100644 --- a/airbyte-integrations/connectors/source-rss/main.py +++ b/airbyte-integrations/connectors/source-rss/main.py @@ -4,5 +4,6 @@ from source_rss.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-rss/source_rss/components.py b/airbyte-integrations/connectors/source-rss/source_rss/components.py index 571d1a0a2e0c..4379d281e143 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/components.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/components.py @@ -12,12 +12,13 @@ import feedparser import pytz import requests +from dateutil.parser import parse + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.types import StreamSlice from airbyte_cdk.sources.streams.core import Stream -from dateutil.parser import parse class CustomExtractor(RecordExtractor): diff --git a/airbyte-integrations/connectors/source-rss/source_rss/run.py b/airbyte-integrations/connectors/source-rss/source_rss/run.py index 90f8a101fcfa..c91bcd0dfb70 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/run.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_rss import SourceRss +from airbyte_cdk.entrypoint import launch + def run(): source = SourceRss() diff --git a/airbyte-integrations/connectors/source-rss/source_rss/source.py b/airbyte-integrations/connectors/source-rss/source_rss/source.py index 297b6a38c9ef..b9e4d0882b56 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/source.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py index 706e9eba88be..de2ec1e2928c 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py @@ -11,6 +11,7 @@ import pytest import yaml + pytest_plugins = ("connector_acceptance_test.plugin",) logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py index 26647e4f62ad..9f4f69e5a062 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py @@ -19,6 +19,9 @@ import orjson import pytest import yaml +from pydantic import BaseModel +from source_s3.v4.source import SourceS3 + from airbyte_cdk.models import ( AirbyteMessage, AirbyteStream, @@ -29,8 +32,7 @@ Type, ) from airbyte_cdk.test import entrypoint_wrapper -from pydantic import BaseModel -from source_s3.v4.source import SourceS3 + if TYPE_CHECKING: from airbyte_cdk import Source @@ -70,6 +72,7 @@ def expect_exception(self) -> bool: def instance_name(self) -> str: return self.config_path.stem + def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: all_tests_config = yaml.safe_load(ACCEPTANCE_TEST_CONFIG_PATH.read_text()) return [ @@ -78,6 +81,7 @@ def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: if "iam_role" not in test["config_path"] ] + # TODO: Convert to a CDK class for better reuse and portability. # class TestSourceAcceptanceTestSuiteBase: # """Test suite for acceptance tests.""" diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py index 6f38722d30cc..2596833e33a4 100644 --- a/airbyte-integrations/connectors/source-s3/main.py +++ b/airbyte-integrations/connectors/source-s3/main.py @@ -4,5 +4,6 @@ from source_s3.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py b/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py index 88908adc107d..135162253839 100644 --- a/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py +++ b/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py @@ -28,6 +28,7 @@ import airbyte as ab from airbyte.secrets import GoogleGSMSecretManager, SecretHandle + AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing" CONNECTOR_NAME = "source-s3" MISSING_ONLY = True diff --git a/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py b/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py index f9b8f6a85523..969c27f609dc 100644 --- a/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py +++ b/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py @@ -30,6 +30,7 @@ from airbyte.secrets import get_secret from airbyte.secrets.google_gsm import GoogleGSMSecretManager, GSMSecretHandle + AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing" CONNECTOR_NAME = "source-s3" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py index 69799dfa2dae..4ad361ac89aa 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py @@ -7,9 +7,11 @@ from traceback import format_exc from typing import Any, List, Mapping, Optional, Tuple -from airbyte_cdk import AbstractSource, ConnectorSpecification, DestinationSyncMode, Stream, SyncMode from wcmatch.glob import GLOBSTAR, SPLIT, globmatch +from airbyte_cdk import AbstractSource, ConnectorSpecification, DestinationSyncMode, Stream, SyncMode + + # ideas on extending this to handle multiple streams: # - "dataset" is currently the name of the single table/stream. We could allow comma-split table names in this string for many streams. # - "path_pattern" currently uses https://facelessuser.github.io/wcmatch/glob/ to match a single string pattern (can be multiple | separated) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py index a623f8ccf4e4..6e206b720a02 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py @@ -15,6 +15,7 @@ from .formats.jsonl_spec import JsonlFormat from .formats.parquet_spec import ParquetFormat + # To implement your provider specific spec, inherit from SourceFilesAbstractSpec and add provider-specific settings e.g.: # class SourceS3Spec(SourceFilesAbstractSpec, BaseModel): diff --git a/airbyte-integrations/connectors/source-s3/source_s3/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/stream.py index b99632dbdc51..6ea5ed89e969 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/stream.py @@ -6,11 +6,12 @@ from typing import Any, Iterator, Mapping import pendulum -from airbyte_cdk import AirbyteTracedException, FailureType from boto3 import session as boto3session from botocore import UNSIGNED from botocore.config import Config from botocore.exceptions import ClientError + +from airbyte_cdk import AirbyteTracedException, FailureType from source_s3.s3_utils import make_s3_client from .s3file import S3File @@ -51,9 +52,7 @@ def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[ # list_objects_v2 doesn't like a None value for ContinuationToken # so we don't set it if we don't have one. if ctoken: - kwargs = dict( - Bucket=provider["bucket"], Prefix=provider.get("path_prefix", ""), ContinuationToken=ctoken - ) # type: ignore[unreachable] + kwargs = dict(Bucket=provider["bucket"], Prefix=provider.get("path_prefix", ""), ContinuationToken=ctoken) # type: ignore[unreachable] else: kwargs = dict(Bucket=provider["bucket"], Prefix=provider.get("path_prefix", "")) try: diff --git a/airbyte-integrations/connectors/source-s3/source_s3/utils.py b/airbyte-integrations/connectors/source-s3/source_s3/utils.py index 9118ec151ff8..f3715bb367a7 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/utils.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/utils.py @@ -11,6 +11,7 @@ import dill import orjson + from airbyte_cdk.models import AirbyteMessage, AirbyteMessageSerializer diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index 5af899aea95c..cdc66ab06fc7 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -5,11 +5,12 @@ from typing import Any, Dict, Optional import dpath.util -from airbyte_cdk import is_cloud_environment -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords from pydantic.v1 import AnyUrl, Field, root_validator from pydantic.v1.error_wrappers import ValidationError +from airbyte_cdk import is_cloud_environment +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords + class Config(AbstractFileBasedSpec): """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py index 0e6cf7528eee..ab60ecf088a1 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.sources.file_based.types import StreamState + logger = logging.Logger("source-S3") diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py index 4d04411a6694..c6db739caf2f 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py @@ -12,6 +12,7 @@ from source_s3.source_files_abstract.formats.jsonl_spec import JsonlFormat from source_s3.source_files_abstract.formats.parquet_spec import ParquetFormat + SECONDS_FORMAT = "%Y-%m-%dT%H:%M:%SZ" MICROS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py index 9991c444ff64..2e3ac647a3a0 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -10,6 +10,7 @@ from typing import Any, Dict, Mapping, Optional import orjson + from airbyte_cdk import ( AirbyteEntrypoint, ConnectorSpecification, @@ -34,6 +35,7 @@ from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer from source_s3.v4.stream_reader import SourceS3StreamReader + _V3_DEPRECATION_FIELD_MAPPING = { "dataset": "streams.name", "format": "streams.format", diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py index aff8c257686d..9c6051c8dd16 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -14,18 +14,20 @@ import psutil import pytz import smart_open -from airbyte_cdk import FailureType -from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError, FileSizeLimitError -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient from botocore.client import Config as ClientConfig from botocore.credentials import RefreshableCredentials from botocore.exceptions import ClientError from botocore.session import get_session +from typing_extensions import override + +from airbyte_cdk import FailureType +from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError, FileSizeLimitError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_s3.v4.config import Config from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler -from typing_extensions import override + AWS_EXTERNAL_ID = getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 4f475b80c797..12d0ed1ff8f0 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -5,10 +5,12 @@ import zipfile from typing import IO, List, Optional, Tuple, Union -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient + +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_s3.v4.config import Config + # Buffer constants BUFFER_SIZE_DEFAULT = 1024 * 1024 MAX_BUFFER_SIZE_DEFAULT: int = 16 * BUFFER_SIZE_DEFAULT diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py index 13e84edd6cb6..1f2e8c726162 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py @@ -9,6 +9,7 @@ from pydantic.v1.error_wrappers import ValidationError from source_s3.v4.config import Config + logger = logging.Logger("") diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py index f06e8fd9ae7c..8b064c65df26 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py @@ -7,11 +7,12 @@ from unittest.mock import Mock import pytest +from source_s3.v4.cursor import Cursor + from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor -from source_s3.v4.cursor import Cursor def _create_datetime(dt: str) -> datetime: diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py index cf4a82f8d3d0..04a23079910b 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py @@ -8,6 +8,7 @@ from source_s3.v4 import Config, SourceS3, SourceS3StreamReader + _V3_FIELDS = ["dataset", "format", "path_pattern", "provider", "schema"] TEST_FILES_FOLDER = Path(__file__).resolve().parent.parent.joinpath("sample_files") diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 02e76e5790ea..2512d5672daa 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -11,16 +11,18 @@ from unittest.mock import ANY, MagicMock, Mock, patch import pytest -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError -from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.stub import Stubber from moto import mock_sts from pydantic.v1 import AnyUrl from source_s3.v4.config import Config from source_s3.v4.stream_reader import SourceS3StreamReader +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile + + logger = logging.Logger("") endpoint_values = ["https://fake.com", None] @@ -124,7 +126,7 @@ def test_get_matching_files( except Exception as exc: raise exc - with patch.object(SourceS3StreamReader, 's3_client', new_callable=MagicMock) as mock_s3_client: + with patch.object(SourceS3StreamReader, "s3_client", new_callable=MagicMock) as mock_s3_client: _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages) files = list(reader.get_matching_files(globs, None, logger)) assert set(f.uri for f in files) == expected_uris @@ -134,27 +136,33 @@ def _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages): responses = [] if multiple_pages and len(mocked_response) > 1: # Split the mocked_response for pagination simulation - first_half = mocked_response[:len(mocked_response) // 2] - second_half = mocked_response[len(mocked_response) // 2:] - - responses.append({ - "IsTruncated": True, - "Contents": first_half, - "KeyCount": len(first_half), - "NextContinuationToken": "token", - }) - - responses.append({ - "IsTruncated": False, - "Contents": second_half, - "KeyCount": len(second_half), - }) + first_half = mocked_response[: len(mocked_response) // 2] + second_half = mocked_response[len(mocked_response) // 2 :] + + responses.append( + { + "IsTruncated": True, + "Contents": first_half, + "KeyCount": len(first_half), + "NextContinuationToken": "token", + } + ) + + responses.append( + { + "IsTruncated": False, + "Contents": second_half, + "KeyCount": len(second_half), + } + ) else: - responses.append({ - "IsTruncated": False, - "Contents": mocked_response, - "KeyCount": len(mocked_response), - }) + responses.append( + { + "IsTruncated": False, + "Contents": mocked_response, + "KeyCount": len(mocked_response), + } + ) def list_objects_v2_side_effect(Bucket, Prefix=None, ContinuationToken=None, **kwargs): if ContinuationToken == "token": @@ -252,7 +260,13 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): mock_s3_client_instance.download_file.return_value = None reader = SourceS3StreamReader() - reader.config = Config(bucket="test", aws_access_key_id="test", aws_secret_access_key="test", streams=[], delivery_method= { "delivery_type": "use_file_transfer" }) + reader.config = Config( + bucket="test", + aws_access_key_id="test", + aws_secret_access_key="test", + streams=[], + delivery_method={"delivery_type": "use_file_transfer"}, + ) try: reader.config = Config( bucket="test", @@ -260,14 +274,14 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): aws_secret_access_key="test", streams=[], endpoint=None, - delivery_method={"delivery_type": "use_file_transfer"} + delivery_method={"delivery_type": "use_file_transfer"}, ) except Exception as exc: raise exc test_file_path = "directory/file.txt" result = reader.get_file(RemoteFile(uri="", last_modified=datetime.now()), test_file_path, logger) - assert result == {'bytes': 100, 'file_relative_path': ANY, 'file_url': ANY} + assert result == {"bytes": 100, "file_relative_path": ANY, "file_url": ANY} assert result["file_url"].endswith(test_file_path) @@ -317,27 +331,20 @@ def test_get_iam_s3_client(boto3_client_mock): # Assertions to validate the s3 client assert s3_client is not None + @pytest.mark.parametrize( "start_date, last_modified_date, expected_result", ( # True when file is new or modified after given start_date - ( - datetime.now() - timedelta(days=180), - datetime.now(), - True - ), + (datetime.now() - timedelta(days=180), datetime.now(), True), ( datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), - True + True, ), # False when file is older than given start_date - ( - datetime.now(), - datetime.now() - timedelta(days=180), - False - ) - ) + (datetime.now(), datetime.now() - timedelta(days=180), False), + ), ) def test_filter_file_by_start_date(start_date: datetime, last_modified_date: datetime, expected_result: bool) -> None: reader = SourceS3StreamReader() @@ -347,7 +354,7 @@ def test_filter_file_by_start_date(start_date: datetime, last_modified_date: dat aws_access_key_id="test", aws_secret_access_key="test", streams=[], - start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ") + start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), ) assert expected_result == reader.is_modified_after_start_date(last_modified_date) diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py index 829350b46560..7372ebf129c4 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py @@ -10,10 +10,12 @@ import pytest import requests_mock +from source_salesforce.source import SourceSalesforce + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.test.catalog_builder import CatalogBuilder -from source_salesforce.source import SourceSalesforce + HERE = Path(__file__).parent _ANY_CATALOG = CatalogBuilder().build() diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py index 1742ef72a923..50926148b631 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py @@ -12,11 +12,13 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder + + HERE = Path(__file__).parent NOTE_CONTENT = "It's the note for integration test" @@ -51,7 +53,11 @@ def stream_name(): @pytest.fixture(scope="module") def stream(input_sandbox_config, stream_name, sf): - return SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE).generate_streams(input_sandbox_config, {stream_name: None}, sf)[0]._legacy_stream + return ( + SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) + .generate_streams(input_sandbox_config, {stream_name: None}, sf)[0] + ._legacy_stream + ) def _encode_content(text): diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py index d03de9ed39e8..d6820c89bfbe 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py @@ -2612,11 +2612,12 @@ import json + result = [] for stream in json.loads(x): stream["stream_descriptor"] = stream.pop("streamDescriptor") stream["stream_state"] = stream.pop("streamState") - y = { + y = { "type": "STREAM", "stream": stream, } diff --git a/airbyte-integrations/connectors/source-salesforce/main.py b/airbyte-integrations/connectors/source-salesforce/main.py index 67536217f497..5ae62f667335 100644 --- a/airbyte-integrations/connectors/source-salesforce/main.py +++ b/airbyte-integrations/connectors/source-salesforce/main.py @@ -5,5 +5,6 @@ from source_salesforce.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index 688962a4f8dd..1317006b402b 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -7,16 +7,18 @@ from typing import Any, List, Mapping, Optional, Tuple import requests # type: ignore[import] +from requests import adapters as request_adapters +from requests.exceptions import RequestException # type: ignore[import] + from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, StreamDescriptor from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.utils import AirbyteTracedException -from requests import adapters as request_adapters -from requests.exceptions import RequestException # type: ignore[import] from .exceptions import TypeSalesforceException from .rate_limiting import SalesforceErrorHandler, default_backoff_handler from .utils import filter_streams_by_criteria + STRING_TYPES = [ "byte", "combobox", diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py index a4fcca0c1e7a..179c45ea65f4 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py @@ -6,9 +6,11 @@ import typing from typing import Optional, Tuple +from requests import HTTPError, codes + from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from requests import HTTPError, codes + if typing.TYPE_CHECKING: from airbyte_cdk.sources import Source diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py index 2fa5db058805..13948c7131de 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py @@ -9,10 +9,12 @@ import backoff import requests +from requests import codes, exceptions # type: ignore[import] + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from requests import codes, exceptions # type: ignore[import] + RESPONSE_CONSUMPTION_EXCEPTIONS = ( # We've had a couple of customers with ProtocolErrors, namely: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index a4236f1ff728..ac9a1af9d0dd 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -8,6 +8,10 @@ import isodate import pendulum +from dateutil.relativedelta import relativedelta +from pendulum.parsing.exceptions import ParserError +from requests import codes, exceptions # type: ignore[import] + from airbyte_cdk.logger import AirbyteLogFormatter from airbyte_cdk.models import ( AirbyteMessage, @@ -30,9 +34,6 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.utils.schema_helpers import InternalConfig from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from dateutil.relativedelta import relativedelta -from pendulum.parsing.exceptions import ParserError -from requests import codes, exceptions # type: ignore[import] from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .streams import ( @@ -46,6 +47,7 @@ RestSalesforceSubStream, ) + _DEFAULT_CONCURRENCY = 10 _MAX_CONCURRENCY = 10 logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 6f0ca9a0f60e..2de1867674cb 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -11,6 +11,9 @@ import pendulum import requests # type: ignore[import] +from pendulum import DateTime # type: ignore[attr-defined] +from requests import exceptions + from airbyte_cdk import ( BearerAuthenticator, CursorPaginationStrategy, @@ -46,13 +49,12 @@ from airbyte_cdk.sources.streams.http import HttpClient, HttpStream, HttpSubStream from airbyte_cdk.sources.types import StreamState from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum import DateTime # type: ignore[attr-defined] -from requests import exceptions from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .availability_strategy import SalesforceAvailabilityStrategy from .rate_limiting import BulkNotSupportedException, SalesforceErrorHandler, default_backoff_handler + # https://stackoverflow.com/a/54517228 CSV_FIELD_SIZE_LIMIT = int(ctypes.c_ulong(-1).value // 2) csv.field_size_limit(CSV_FIELD_SIZE_LIMIT) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 1a59386a1b95..a571cada4f53 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -13,6 +13,20 @@ import freezegun import pytest import requests_mock +from config_builder import ConfigBuilder +from conftest import generate_stream +from salesforce_job_response_builder import JobInfoResponseBuilder +from source_salesforce.api import Salesforce +from source_salesforce.source import SourceSalesforce +from source_salesforce.streams import ( + CSV_FIELD_SIZE_LIMIT, + BulkIncrementalSalesforceStream, + BulkSalesforceStream, + BulkSalesforceSubStream, + IncrementalRestSalesforceStream, + RestSalesforceStream, +) + from airbyte_cdk.models import ( AirbyteStateBlob, AirbyteStream, @@ -28,19 +42,7 @@ from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils import AirbyteTracedException -from config_builder import ConfigBuilder -from conftest import generate_stream -from salesforce_job_response_builder import JobInfoResponseBuilder -from source_salesforce.api import Salesforce -from source_salesforce.source import SourceSalesforce -from source_salesforce.streams import ( - CSV_FIELD_SIZE_LIMIT, - BulkIncrementalSalesforceStream, - BulkSalesforceStream, - BulkSalesforceSubStream, - IncrementalRestSalesforceStream, - RestSalesforceStream, -) + _A_CHUNKED_RESPONSE = [b"first chunk", b"second chunk"] _A_JSON_RESPONSE = {"id": "any id"} @@ -100,9 +102,7 @@ def test_stream_slice_step_validation(stream_slice_step: str, expected_error_mes ), ], ) -def test_login_authentication_error_handler( - stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg -): +def test_login_authentication_error_handler(stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg): source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) logger = logging.getLogger("airbyte") requests_mock.register_uri( @@ -557,13 +557,21 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True) job_id_1 = "fake_job_1" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}", [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_1}", + [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_1}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}/results", text="Field1,LastModifiedDate,ID\ntest,2023-01-15,1") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_1}") job_id_2 = "fake_job_2" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_2}", [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_2}", + [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_2}") requests_mock.register_uri( "GET", _bulk_stream_path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22" @@ -574,7 +582,11 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api queries_history = requests_mock.register_uri( "POST", _bulk_stream_path(), [{"json": {"id": job_id_1}}, {"json": {"id": job_id_2}}, {"json": {"id": job_id_3}}] ) - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}", [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_3}", + [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_3}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_3}") @@ -586,18 +598,24 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api # assert request params: has requests might not be performed in a specific order because of concurrent CDK, we match on any request all_requests = {request.text for request in queries_history.request_history} - assert any([ - "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" - in request for request in all_requests - ]) + assert any( + [ + "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" in request + for request in all_requests + ] + ) # as the execution is concurrent, we can only assert the last state message here last_actual_state = [item.state.stream.stream_state for item in result if item.type == Type.STATE][-1] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index 92df1b20876e..348d2de1b081 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -7,13 +7,15 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalogSerializer -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from config_builder import ConfigBuilder from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalogSerializer +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + _ANY_CATALOG = CatalogBuilder().build() _ANY_CONFIG = ConfigBuilder().build() _ANY_STATE = StateBuilder().build() diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py index a5156457d1f2..1aa5376552cb 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py @@ -8,15 +8,17 @@ from unittest import TestCase import freezegun +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from salesforce_job_response_builder import JobCreateResponseBuilder, JobInfoResponseBuilder +from source_salesforce.streams import BulkSalesforceStream + from airbyte_cdk.models import AirbyteStreamStatus, SyncMode from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse -from config_builder import ConfigBuilder from integration.test_rest_stream import create_http_request as create_standard_http_request from integration.test_rest_stream import create_http_response as create_standard_http_response from integration.utils import create_base_url, given_authentication, given_stream, read -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from salesforce_job_response_builder import JobCreateResponseBuilder, JobInfoResponseBuilder -from source_salesforce.streams import BulkSalesforceStream + _A_FIELD_NAME = "a_field" _ANOTHER_FIELD_NAME = "another_field" @@ -25,7 +27,9 @@ _CLIENT_SECRET = "a_client_secret" _CURSOR_FIELD = "SystemModstamp" _INCREMENTAL_FIELDS = [_A_FIELD_NAME, _CURSOR_FIELD] -_INCREMENTAL_SCHEMA_BUILDER = SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") # re-using same fields as _INCREMENTAL_FIELDS +_INCREMENTAL_SCHEMA_BUILDER = ( + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") +) # re-using same fields as _INCREMENTAL_FIELDS _INSTANCE_URL = "https://instance.salesforce.com" _JOB_ID = "a-job-id" _ANOTHER_JOB_ID = "another-job-id" @@ -60,17 +64,16 @@ def _calculate_start_time(start_time: datetime) -> datetime: def _build_job_creation_request(query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps({"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"}), + ) def _make_sliced_job_request(lower_boundary: datetime, upper_boundary: datetime, fields: List[str]) -> HttpRequest: - return _build_job_creation_request(f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + return _build_job_creation_request( + f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(fields: List[str]) -> HttpRequest: @@ -78,7 +81,6 @@ def _make_full_job_request(fields: List[str]) -> HttpRequest: class BulkStreamTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN) @@ -168,7 +170,9 @@ def test_given_type_when_read_then_field_is_casted_with_right_type(self) -> None @freezegun.freeze_time(_NOW.isoformat()) def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -192,7 +196,9 @@ def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: @freezegun.freeze_time(_NOW.isoformat()) def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -220,7 +226,9 @@ def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) @freezegun.freeze_time(_NOW.isoformat()) def test_given_specific_encoding_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -338,7 +346,17 @@ def test_given_non_transient_error_on_job_creation_when_read_then_fail_sync(self given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME]), - HttpResponse(json.dumps([{"errorCode": "API_ERROR", "message": "Implementation restriction... "}]), 400), + HttpResponse( + json.dumps( + [ + { + "errorCode": "API_ERROR", + "message": "Implementation restriction... ", + } + ] + ), + 400, + ), ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -526,11 +544,21 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) self._config.start_date(start_date).stream_slice_step("P7D") given_stream(self._http_mocker, _BASE_URL, _STREAM_WITH_PARENT_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) - self._create_sliced_job_with_records(start_date, first_upper_boundary, _PARENT_STREAM_NAME, "first_parent_slice_job_id", [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}]) - self._create_sliced_job_with_records(first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}]) + self._create_sliced_job_with_records( + start_date, + first_upper_boundary, + _PARENT_STREAM_NAME, + "first_parent_slice_job_id", + [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}], + ) + self._create_sliced_job_with_records( + first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}] + ) self._http_mocker.post( - self._build_job_creation_request(f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')"), + self._build_job_creation_request( + f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')" + ), JobCreateResponseBuilder().with_id(_JOB_ID).build(), ) self._http_mocker.get( @@ -547,10 +575,16 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) assert len(output.records) == 1 - def _create_sliced_job(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int) -> None: - self._create_sliced_job_with_records(lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count)) + def _create_sliced_job( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int + ) -> None: + self._create_sliced_job_with_records( + lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count) + ) - def _create_sliced_job_with_records(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]]) -> None: + def _create_sliced_job_with_records( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]] + ) -> None: self._http_mocker.post( self._make_sliced_job_request(lower_boundary, upper_boundary, stream_name, list(records[0].keys())), JobCreateResponseBuilder().with_id(job_id).build(), @@ -581,8 +615,12 @@ def _create_csv(self, headers: List[str], data: List[Dict[str, str]], dialect: s writer.writerow(line) return csvfile.getvalue() - def _make_sliced_job_request(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str]) -> HttpRequest: - return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + def _make_sliced_job_request( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str] + ) -> HttpRequest: + return self._build_job_creation_request( + f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(self, fields: List[str], stream_name: str = _STREAM_NAME) -> HttpRequest: return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name}") @@ -601,14 +639,13 @@ def _generate_csv(self, records: List[Dict[str, str]]) -> str: for record in records: csv_entry.append(",".join([record[key] for key in keys])) - entries = '\n'.join(csv_entry) + entries = "\n".join(csv_entry) return f"{','.join(keys)}\n{entries}" def _build_job_creation_request(self, query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps( + {"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"} + ), + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py index 85805d29cb74..d61328f9c5ba 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py @@ -7,15 +7,17 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import AirbyteStateBlob, SyncMode -from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse -from airbyte_cdk.test.state_builder import StateBuilder from config_builder import ConfigBuilder -from integration.utils import create_base_url, given_authentication, given_stream, read from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder from source_salesforce.api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS from source_salesforce.streams import LOOKBACK_SECONDS +from airbyte_cdk.models import AirbyteStateBlob, SyncMode +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.state_builder import StateBuilder +from integration.utils import create_base_url, given_authentication, given_stream, read + + _A_FIELD_NAME = "a_field" _CLIENT_ID = "a_client_id" _CLIENT_SECRET = "a_client_secret" @@ -31,7 +33,7 @@ def create_http_request(stream_name: str, field_names: List[str], access_token: Optional[str] = None) -> HttpRequest: return HttpRequest( f"{_BASE_URL}/queryAll?q=SELECT+{','.join(field_names)}+FROM+{stream_name}+", - headers={"Authorization": f"Bearer {access_token}"} if access_token else None + headers={"Authorization": f"Bearer {access_token}"} if access_token else None, ) @@ -40,7 +42,10 @@ def create_http_response(field_names: List[str], record_count: int = 1) -> HttpR This method does not handle field types for now which may cause some test failures on change if we start considering using some fields for calculation. One example of that would be cursor field parsing to datetime. """ - records = [{field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} for i in range(record_count)] + records = [ + {field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} + for i in range(record_count) + ] return HttpResponse(json.dumps({"records": records})) @@ -64,7 +69,6 @@ def _calculate_start_time(start_time: datetime) -> datetime: @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN) @@ -77,7 +81,7 @@ def test_given_error_on_fetch_chunk_of_properties_when_read_then_retry(self, htt [ HttpResponse("", status_code=406), create_http_response([_A_FIELD_NAME], record_count=1), - ] + ], ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -94,7 +98,12 @@ def setUp(self) -> None: self._http_mocker.__enter__() given_authentication(self._http_mocker, _CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN, _INSTANCE_URL) - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime")) + given_stream( + self._http_mocker, + _BASE_URL, + _STREAM_NAME, + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime"), + ) def tearDown(self) -> None: self._http_mocker.__exit__(None, None, None) @@ -102,11 +111,13 @@ def tearDown(self) -> None: def test_given_no_state_when_read_then_start_sync_from_start(self) -> None: start = _calculate_start_time(_NOW - timedelta(days=5)) # as the start comes from the config, we can't use the same format as `_to_url` - start_format_url = urllib.parse.quote_plus(start.strftime('%Y-%m-%dT%H:%M:%SZ')) + start_format_url = urllib.parse.quote_plus(start.strftime("%Y-%m-%dT%H:%M:%SZ")) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME], record_count=1), ) @@ -119,13 +130,22 @@ def test_given_sequential_state_when_read_then_migrate_to_partitioned_state(self start = _calculate_start_time(_NOW - timedelta(days=10)) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) - output = read(_STREAM_NAME, SyncMode.incremental, self._config, StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")})) + output = read( + _STREAM_NAME, + SyncMode.incremental, + self._config, + StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")}), + ) - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_update_state(self) -> None: missing_chunk = (_NOW - timedelta(days=5), _NOW - timedelta(days=3)) @@ -138,21 +158,27 @@ def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_upda "slices": [ {"start": start.strftime("%Y-%m-%dT%H:%M:%S.000") + "Z", "end": _to_partitioned_datetime(missing_chunk[0])}, {"start": _to_partitioned_datetime(missing_chunk[1]), "end": _to_partitioned_datetime(most_recent_state_value)}, - ] - } + ], + }, ) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) output = read(_STREAM_NAME, SyncMode.incremental, self._config, state) # the start is granular to the second hence why we have `000` in terms of milliseconds - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py index 3fa57dfbd0a1..cd380be8ee2c 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py @@ -4,15 +4,17 @@ from unittest import TestCase import pytest +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from source_salesforce import SourceSalesforce + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from config_builder import ConfigBuilder from integration.utils import create_base_url, given_authentication, given_stream -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from source_salesforce import SourceSalesforce + _CLIENT_ID = "a_client_id" _CLIENT_SECRET = "a_client_secret" @@ -25,13 +27,10 @@ class StreamGenerationTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN).build() self._source = SourceSalesforce( - CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), - self._config, - StateBuilder().build() + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), self._config, StateBuilder().build() ) self._http_mocker = HttpMocker() @@ -48,10 +47,7 @@ def test_given_transient_error_fetching_schema_when_streams_then_retry(self) -> ) self._http_mocker.get( HttpRequest(f"{_BASE_URL}/sobjects/{_STREAM_NAME}/describe"), - [ - HttpResponse("", status_code=406), - SalesforceDescribeResponseBuilder().field("a_field_name").build() - ] + [HttpResponse("", status_code=406), SalesforceDescribeResponseBuilder().field("a_field_name").build()], ) streams = self._source.streams(self._config) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py index b547a297c439..bae6d4bd1445 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py @@ -3,6 +3,10 @@ import json from typing import Any, Dict, Optional +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from source_salesforce import SourceSalesforce + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -11,9 +15,7 @@ from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from source_salesforce import SourceSalesforce + _API_VERSION = "v57.0" @@ -35,7 +37,7 @@ def read( sync_mode: SyncMode, config_builder: Optional[ConfigBuilder] = None, state_builder: Optional[StateBuilder] = None, - expecting_exception: bool = False + expecting_exception: bool = False, ) -> EntrypointOutput: catalog = _catalog(stream_name, sync_mode) config = config_builder.build() if config_builder else ConfigBuilder().build() @@ -43,12 +45,19 @@ def read( return entrypoint_read(_source(catalog, config, state), config, catalog, state, expecting_exception) -def given_authentication(http_mocker: HttpMocker, client_id: str, client_secret: str, refresh_token: str, instance_url: str, access_token: str = "any_access_token") -> None: +def given_authentication( + http_mocker: HttpMocker, + client_id: str, + client_secret: str, + refresh_token: str, + instance_url: str, + access_token: str = "any_access_token", +) -> None: http_mocker.post( HttpRequest( "https://login.salesforce.com/services/oauth2/token", query_params=ANY_QUERY_PARAMS, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}", ), HttpResponse(json.dumps({"access_token": access_token, "instance_url": instance_url})), ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py index 55bc8b8f65dd..76af9b7eaf38 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py @@ -9,18 +9,18 @@ class JobCreateResponseBuilder: def __init__(self) -> None: self._response = { - "id": "any_id", - "operation": "query", - "object": "Account", - "createdById": "005R0000000GiwjIAC", - "createdDate": "2018-12-17T21:00:17.000+0000", - "systemModstamp": "2018-12-17T21:00:17.000+0000", - "state": "UploadComplete", - "concurrencyMode": "Parallel", - "contentType": "CSV", - "apiVersion": 46.0, - "lineEnding": "LF", - "columnDelimiter": "COMMA" + "id": "any_id", + "operation": "query", + "object": "Account", + "createdById": "005R0000000GiwjIAC", + "createdDate": "2018-12-17T21:00:17.000+0000", + "systemModstamp": "2018-12-17T21:00:17.000+0000", + "state": "UploadComplete", + "concurrencyMode": "Parallel", + "contentType": "CSV", + "apiVersion": 46.0, + "lineEnding": "LF", + "columnDelimiter": "COMMA", } self._status_code = 200 @@ -52,11 +52,11 @@ def with_state(self, state: str) -> "JobInfoResponseBuilder": def with_status_code(self, status_code: int) -> "JobInfoResponseBuilder": self._status_code = status_code return self - + def with_error_message(self, error_message: str) -> "JobInfoResponseBuilder": self._response["errorMessage"] = error_message return self - + def get_response(self) -> any: return self._response diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py index 2c8abd44f859..cd00f723774a 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py @@ -4,10 +4,12 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.streams import Stream from requests import HTTPError, Response from source_salesforce.availability_strategy import SalesforceAvailabilityStrategy +from airbyte_cdk.sources.streams import Stream + + _NO_SOURCE = None diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py index c6b9d1ea1731..c00beb8981c1 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py @@ -6,11 +6,13 @@ import pytest import requests import requests_mock -from airbyte_cdk.models import FailureType -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from requests.exceptions import ChunkedEncodingError from source_salesforce.rate_limiting import BulkNotSupportedException, SalesforceErrorHandler +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + + _ANY = "any" _ANY_BASE_URL = "https://any-base-url.com" _SF_API_VERSION = "v57.0" @@ -20,18 +22,32 @@ class SalesforceErrorHandlerTest(TestCase): def setUp(self) -> None: self._error_handler = SalesforceErrorHandler() - def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}]) + def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported( + self, + ) -> None: + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_compound_data_error_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_request_limit_exceeded_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) @@ -41,12 +57,24 @@ def test_given_limit_exceeded_on_job_creation_when_interpret_response_then_raise self._error_handler.interpret_response(response) def test_given_query_not_supported_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}]) + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_txn_security_metering_error_when_interpret_response_then_raise_config_error(self) -> None: - response = self._create_response("GET", self._url_for_job_creation() + "/job_id", 400, [{"errorCode": "TXN_SECURITY_METERING_ERROR", "message": "We can't complete the action because enabled transaction security policies took too long to complete."}]) + response = self._create_response( + "GET", + self._url_for_job_creation() + "/job_id", + 400, + [ + { + "errorCode": "TXN_SECURITY_METERING_ERROR", + "message": "We can't complete the action because enabled transaction security policies took too long to complete.", + } + ], + ) error_resolution = self._error_handler.interpret_response(response) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py index 25ea70f6362c..b2f335f594e8 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py @@ -4,20 +4,24 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import SyncMode from config_builder import ConfigBuilder from conftest import generate_stream, mock_stream_api from source_salesforce.api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS +from airbyte_cdk.models import SyncMode + + _NOW = datetime.fromisoformat("2020-01-01T00:00:00+00:00") _STREAM_NAME = UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] + @freezegun.freeze_time(time_to_freeze=_NOW) class IncrementalSliceGenerationTest(TestCase): """ For this, we will be testing with UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] as bulk stream slicing actually creates jobs. We will assume the bulk one usese the same logic. """ + def test_given_start_within_slice_range_when_stream_slices_then_return_one_slice_considering_10_minutes_lookback(self) -> None: config = ConfigBuilder().start_date(_NOW - timedelta(days=15)).stream_slice_step("P30D").build() stream = generate_stream(_STREAM_NAME, config, mock_stream_api(config)) @@ -34,5 +38,5 @@ def test_given_slice_range_smaller_than_now_minus_start_date_when_stream_slices_ assert slices == [ {"start_date": "2019-11-22T00:00:00.000+00:00", "end_date": "2019-12-22T00:00:00.000+00:00"}, - {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"} + {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"}, ] diff --git a/airbyte-integrations/connectors/source-salesloft/components.py b/airbyte-integrations/connectors/source-salesloft/components.py index 5475aa13ca33..4d94758110e5 100644 --- a/airbyte-integrations/connectors/source-salesloft/components.py +++ b/airbyte-integrations/connectors/source-salesloft/components.py @@ -10,7 +10,6 @@ @dataclass class SingleUseOauth2Authenticator(DeclarativeSingleUseRefreshTokenOauth2Authenticator): - config: Config def __post_init__(self): diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sentry/main.py b/airbyte-integrations/connectors/source-sentry/main.py index 1c7adc746e97..c5f41c2a4502 100644 --- a/airbyte-integrations/connectors/source-sentry/main.py +++ b/airbyte-integrations/connectors/source-sentry/main.py @@ -4,5 +4,6 @@ from source_sentry.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py index 40282ee1ff62..44ad79700b2f 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py @@ -8,9 +8,10 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_sentry import SourceSentry diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py index 0c5af692c5af..ab243cf755d6 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "auth_token": "test token", "organization": "test organization", "project": "test project", - "hostname": "sentry.io" + "hostname": "sentry.io", } def build(self) -> Dict[str, Any]: diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py index 85987dd3d2dd..3c29f713d5dd 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py @@ -3,14 +3,15 @@ import json from unittest import TestCase +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_sentry.source import SourceSentry class TestEvents(TestCase): @@ -29,12 +30,8 @@ def state(self): @HttpMocker() def test_read(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() @@ -47,12 +44,8 @@ def test_read(self, http_mocker: HttpMocker): @HttpMocker() def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py index 0103ab4856e8..40c869236242 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py @@ -3,14 +3,15 @@ import json from unittest import TestCase +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_sentry.source import SourceSentry class TestEvents(TestCase): @@ -31,10 +32,9 @@ def test_read(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) # https://sentry.io/api/1/projects/airbyte-09/airbyte-09/issues/?query=lastSeen%3A%3E2022-01-01T00%3A00%3A00.0Z config = self.config() @@ -50,10 +50,9 @@ def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 5f3c1f349ebd..0bab2123c0fd 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -5,9 +5,11 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from source_sentry import SourceSentry +from airbyte_cdk.models import SyncMode + + INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} @@ -24,7 +26,10 @@ def test_next_page_token(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": "next-page"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == "next-page" + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == "next-page" + ) def test_next_page_token_is_none(): @@ -33,7 +38,9 @@ def test_next_page_token_is_none(): response_mock.headers = {} # stop condition: "results": "false" response_mock.links = {"next": {"cursor": "", "results": "false"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + ) def test_events_path(): @@ -77,7 +84,10 @@ def test_projects_request_params(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": expected}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == expected + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == expected + ) def test_project_detail_request_params(): @@ -89,10 +99,7 @@ def test_project_detail_request_params(): def test_project_detail_parse_response(requests_mock): expected = {"id": "1", "name": "test project"} stream = get_stream_by_name("project_detail") - requests_mock.get( - "https://sentry.io/api/0/projects/test-org/test-project/", - json=expected - ) + requests_mock.get("https://sentry.io/api/0/projects/test-org/test-project/", json=expected) result = list(stream.read_records(sync_mode=SyncMode.full_refresh))[0] assert expected == result.data @@ -137,4 +144,3 @@ def test_issues_validate_state_value(state, expected): stream = get_stream_by_name("issues") stream.retriever.state = state assert stream.state.get(stream.cursor_field) == expected - diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py index d2f090c60286..75e813d64e9a 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py @@ -11,6 +11,7 @@ import docker import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) TMP_FOLDER = "/tmp/test_sftp_source" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py index 5278db267df5..fb90225ddeb1 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py @@ -14,10 +14,12 @@ import docker import paramiko import pytest + from airbyte_cdk import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from .utils import get_docker_ip, load_config + logger = logging.getLogger("airbyte") PRIVATE_KEY = str() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py index f6e0b7560c29..1eb360614d76 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py @@ -10,10 +10,12 @@ from unittest.mock import ANY import pytest +from source_sftp_bulk import SourceSFTPBulk + from airbyte_cdk import AirbyteTracedException, ConfiguredAirbyteCatalog, Status from airbyte_cdk.sources.declarative.models import FailureType from airbyte_cdk.test.entrypoint_wrapper import read -from source_sftp_bulk import SourceSFTPBulk + logger = logging.getLogger("airbyte") @@ -99,18 +101,26 @@ def test_get_files_empty_files(configured_catalog: ConfiguredAirbyteCatalog, con def test_get_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_file_transfer: Mapping[str, Any]): source = SourceSFTPBulk(catalog=configured_catalog, config=config_fixture_use_file_transfer, state=None) output = read(source=source, config=config_fixture_use_file_transfer, catalog=configured_catalog) - expected_file_data = {'bytes': 46_754_266, 'file_relative_path': 'files/file_transfer/file_transfer_1.csv', 'file_url': '/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv', 'modified': ANY, 'source_file_url': '/files/file_transfer/file_transfer_1.csv'} + expected_file_data = { + "bytes": 46_754_266, + "file_relative_path": "files/file_transfer/file_transfer_1.csv", + "file_url": "/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv", + "modified": ANY, + "source_file_url": "/files/file_transfer/file_transfer_1.csv", + } assert len(output.records) == 1 assert list(map(lambda record: record.record.file, output.records)) == [expected_file_data] # Additional assertion to check if the file exists at the file_url path - file_path = expected_file_data['file_url'] + file_path = expected_file_data["file_url"] assert os.path.exists(file_path), f"File not found at path: {file_path}" @pytest.mark.slow @pytest.mark.limit_memory("10 MB") -def test_get_all_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any]): +def test_get_all_file_csv_file_transfer( + configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any] +): """ - The Paramiko dependency `get` method uses requests parallelization for efficiency, which may slightly increase memory usage. - The test asserts that this memory increase remains below the files sizes being transferred. diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py index db92623c2998..390cb801cdb7 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py @@ -7,6 +7,7 @@ import re from typing import Any, Mapping, Union + logger = logging.getLogger("airbyte") TMP_FOLDER = "/tmp/test_sftp_source" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py index 9129b1995968..f2799e1acdf7 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/main.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/main.py @@ -5,5 +5,6 @@ from source_sftp_bulk.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py index 8a18fac42f1e..1f6e04360fc1 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py @@ -9,9 +9,11 @@ import backoff import paramiko -from airbyte_cdk import AirbyteTracedException, FailureType from paramiko.ssh_exception import AuthenticationException +from airbyte_cdk import AirbyteTracedException, FailureType + + # set default timeout to 300 seconds REQUEST_TIMEOUT = 300 diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py index 96b84eda4c94..5a10c33cfc9d 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py @@ -12,6 +12,7 @@ from source_sftp_bulk.spec import SourceSFTPBulkSpec from source_sftp_bulk.stream_reader import SourceSFTPBulkStreamReader + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py index 06863da1ae01..23edbcf1ebf0 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py @@ -3,9 +3,10 @@ from typing import Literal, Optional, Union +from pydantic.v1 import BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords -from pydantic.v1 import BaseModel, Field class PasswordCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py index 10d075f001e6..d3f0ae7c525a 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py @@ -9,13 +9,14 @@ from typing import Dict, Iterable, List, Optional import psutil +from typing_extensions import override + from airbyte_cdk import FailureType from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_sftp_bulk.client import SFTPClient from source_sftp_bulk.spec import SourceSFTPBulkSpec -from typing_extensions import override class SourceSFTPBulkStreamReader(AbstractFileBasedStreamReader): diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py index 1fa424d9d1a8..92c81a790b04 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py @@ -10,6 +10,7 @@ from source_sftp_bulk.spec import SourceSFTPBulkSpec from source_sftp_bulk.stream_reader import SourceSFTPBulkStreamReader + logger = logging.Logger("") diff --git a/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-shopify/main.py b/airbyte-integrations/connectors/source-shopify/main.py index aca13eebbb25..2bfe3cb0cfe9 100644 --- a/airbyte-integrations/connectors/source-shopify/main.py +++ b/airbyte-integrations/connectors/source-shopify/main.py @@ -5,5 +5,6 @@ from source_shopify.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py index 6d53197ff18a..3d955fd8b5ec 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -25,7 +25,6 @@ def __init__(self, auth_method: str = None): class ShopifyAuthenticator(TokenAuthenticator): - """ Making Authenticator to be able to accept Header-Based authentication. """ diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py b/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py index eba59a079355..412c0e3b05a3 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py @@ -5,12 +5,13 @@ from typing import Any, List, Mapping +from orjson import orjson + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import AirbyteMessageSerializer from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository -from orjson import orjson class MigrateConfig: diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py index d4e4f6b728dc..873432ef3084 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py @@ -3,9 +3,11 @@ from typing import Optional, Union import requests +from requests import exceptions + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction -from requests import exceptions + RESPONSE_CONSUMPTION_EXCEPTIONS = ( exceptions.ChunkedEncodingError, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py index 1a493119b901..33c1340133c4 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py @@ -7,12 +7,14 @@ from typing import Any, Iterable, List, Mapping, Optional import requests -from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import InvalidURL, JSONDecodeError +from airbyte_cdk.sources.streams.http import HttpClient + from .http_request import ShopifyErrorHandler from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyWrongShopNameError + SCOPES_MAPPING: Mapping[str, set[str]] = { # SCOPE: read_customers "Customers": ("read_customers",), @@ -82,7 +84,6 @@ class ShopifyScopes: - # define default logger logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py index a479bf68ff5a..0034d87ae20b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py @@ -9,11 +9,12 @@ import pendulum as pdm import requests -from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import JSONDecodeError from source_shopify.utils import LOGGER, ApiTypeEnum from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.sources.streams.http import HttpClient + from .exceptions import AirbyteTracedException, ShopifyBulkExceptions from .query import ShopifyBulkQuery, ShopifyBulkTemplates from .record import ShopifyBulkRecord diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py index 7c249196d928..a1bfcc676645 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py @@ -2562,7 +2562,6 @@ def _should_include_presentment_prices(self) -> bool: @property def query_nodes(self) -> Optional[Union[List[Field], List[str]]]: - prices_fields: List[str] = ["amount", "currencyCode"] presentment_prices_fields: List[Field] = [ Field( diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py index 61e80937a354..ffa0852de799 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py @@ -8,6 +8,7 @@ from .exceptions import ShopifyBulkExceptions + BULK_RETRY_ERRORS: Final[Tuple] = ( ShopifyBulkExceptions.BulkJobBadResponse, ShopifyBulkExceptions.BulkJobError, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py index dfa3fafdd0c6..40fa1e8b2733 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py @@ -11,6 +11,7 @@ from .exceptions import ShopifyBulkExceptions + # default end line tag END_OF_FILE: str = "" BULK_PARENT_KEY: str = "__parentId" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py index 462ad3ea3aa8..d7a7f6b4a76e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py @@ -9,6 +9,7 @@ from . import schema + _schema = schema _schema_root = _schema.shopify_schema diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py index d3647a562084..fb24170f11be 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py @@ -7,6 +7,7 @@ import sgqlc.types.datetime import sgqlc.types.relay + shopify_schema = sgqlc.types.Schema() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 3d420b331891..76a2e27f83c1 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -6,11 +6,12 @@ import logging from typing import Any, List, Mapping, Tuple +from requests.exceptions import ConnectionError, RequestException, SSLError + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from requests.exceptions import ConnectionError, RequestException, SSLError from .auth import MissingAccessTokenError, ShopifyAuthenticator from .scopes import ShopifyScopes diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py index 3837212ad9f2..fc5ead602a7c 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py @@ -12,11 +12,6 @@ import pendulum as pdm import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpClient, HttpStream -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler -from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from requests.exceptions import RequestException from source_shopify.http_request import ShopifyErrorHandler from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager @@ -26,6 +21,12 @@ from source_shopify.utils import ShopifyNonRetryableErrors from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.streams.http import HttpClient, HttpStream +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler +from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING + class ShopifyStream(HttpStream, ABC): # define default logger diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py index 2751a3ab9756..58f4986221c6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py @@ -6,8 +6,6 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from requests.exceptions import RequestException from source_shopify.shopify_graphql.bulk.query import ( Collection, @@ -36,6 +34,9 @@ from source_shopify.utils import ApiTypeEnum from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader + from .base_streams import ( IncrementalShopifyGraphQlBulkStream, IncrementalShopifyNestedStream, @@ -249,7 +250,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream): class BalanceTransactions(IncrementalShopifyStream): - """ PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: https://shopify.dev/api/admin-rest/2021-07/resources/transactions diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index d7fd17a42846..fa510b87ba91 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -10,10 +10,12 @@ from typing import Any, Callable, Dict, Final, List, Mapping, Optional import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction from airbyte_cdk.utils import AirbyteTracedException + # default logger instance LOGGER: Final[logging.Logger] = logging.getLogger("airbyte") @@ -47,7 +49,7 @@ def __new__(self, stream: str) -> Mapping[str, Any]: response_action=ResponseAction.IGNORE, failure_type=FailureType.config_error, error_message=f"Stream `{stream}`. Entity might not be available or missing.", - ) + ), # extend the mapping with more handable errors, if needed. } diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index 0f296c87061a..4dbba74a333b 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -9,8 +9,10 @@ import pytest import requests + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" @@ -39,7 +41,7 @@ def logger(): @pytest.fixture def basic_config(): return { - "shop": "test_shop", + "shop": "test_shop", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "shop_id": 0, } @@ -52,7 +54,6 @@ def auth_config(): "start_date": "2023-01-01", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "authenticator": None, - } @@ -358,7 +359,7 @@ def bulk_job_failed_response(): }, } - + @pytest.fixture def bulk_job_failed_with_partial_url_response(): return { @@ -371,20 +372,16 @@ def bulk_job_failed_with_partial_url_response(): "fileSize": None, "url": None, "partialDataUrl": 'https://some_url?response-content-disposition=attachment;+filename="bulk-123456789.jsonl";+filename*=UTF-8' - "bulk-123456789.jsonl&response-content-type=application/jsonl" + "bulk-123456789.jsonl&response-content-type=application/jsonl", } }, "extensions": { "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 20000.0, - "currentlyAvailable": 19999, - "restoreRate": 1000.0 - } + "throttleStatus": {"maximumAvailable": 20000.0, "currentlyAvailable": 19999, "restoreRate": 1000.0}, } - } + }, } @@ -442,8 +439,8 @@ def bulk_job_running_response(): } }, } - - + + @pytest.fixture def bulk_job_running_with_object_count_and_url_response(): return { diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py index 90eb6faaf67d..afce4d33ab18 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py @@ -7,7 +7,6 @@ import pytest import requests -from airbyte_cdk.models import SyncMode from source_shopify.shopify_graphql.bulk.exceptions import ShopifyBulkExceptions from source_shopify.shopify_graphql.bulk.status import ShopifyBulkJobStatus from source_shopify.streams.streams import ( @@ -25,15 +24,18 @@ TransactionsGraphql, ) +from airbyte_cdk.models import SyncMode + + _ANY_SLICE = {} _ANY_FILTER_FIELD = "any_filter_field" def test_job_manager_default_values(auth_config) -> None: stream = Products(auth_config) - + # 10Mb chunk size to save the file - assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 + assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 assert stream.job_manager._job_max_retries == 6 assert stream.job_manager._job_backoff_time == 5 # running job logger constrain, every 100-ish message will be printed @@ -48,7 +50,7 @@ def test_job_manager_default_values(auth_config) -> None: # currents: _job_id, _job_state, _job_created_at, _job_self_canceled assert not stream.job_manager._job_id # this string is based on ShopifyBulkJobStatus - assert not stream.job_manager._job_state + assert not stream.job_manager._job_state # completed and saved Bulk Job result filename assert not stream.job_manager._job_result_filename # date-time when the Bulk Job was created on the server @@ -56,7 +58,7 @@ def test_job_manager_default_values(auth_config) -> None: # indicated whether or not we manually force-cancel the current job assert not stream.job_manager._job_self_canceled # time between job status checks - assert stream.job_manager. _job_check_interval == 3 + assert stream.job_manager._job_check_interval == 3 # 0.1 ~= P2H, default value, lower boundary for slice size assert stream.job_manager._job_size_min == 0.1 # last running job object count @@ -99,8 +101,9 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: {"json": request.getfixturevalue("bulk_error_with_concurrent_job")}, # concurrent request has finished {"json": request.getfixturevalue("bulk_successful_response")}, - ]) - + ], + ) + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) # call count should be 4 (3 retries, 1 - succeeded) assert requests_mock.call_count == 4 @@ -119,14 +122,16 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: ], ids=[ "max attempt reached", - ] + ], ) -def test_job_retry_on_concurrency(request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected) -> None: +def test_job_retry_on_concurrency( + request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # patching concurrent settings stream.job_manager._concurrent_max_retry = concurrent_max_retry stream.job_manager._concurrent_interval = 1 - + requests_mock.post(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) if error_type: @@ -200,8 +205,8 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, mocker.patch("source_shopify.shopify_graphql.bulk.record.ShopifyBulkRecord.read_file", return_value=[]) stream.job_manager._job_check_state() assert expected == stream.job_manager._job_result_filename - - + + @pytest.mark.parametrize( "job_response, error_type, expected", [ @@ -211,7 +216,9 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, "failed", ], ) -def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, requests_mock, job_response, error_type, expected, auth_config) -> None: +def test_job_failed_for_stream_with_no_bulk_checkpointing( + mocker, request, requests_mock, job_response, error_type, expected, auth_config +) -> None: stream = InventoryLevels(auth_config) # modify the sleep time for the test stream.job_manager._concurrent_max_retry = 1 @@ -253,26 +260,28 @@ def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, reque "BulkJobBadResponse", ], ) -def test_retry_on_job_creation_exception(request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg) -> None: +def test_retry_on_job_creation_exception( + request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg +) -> None: stream = MetafieldOrders(auth_config) stream.job_manager._job_backoff_time = 0 stream.job_manager._job_max_retries = max_retry # patching the method to get the right ID checks if job_response: stream.job_manager._job_id = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("id") - + if job_state: # setting job_state to simulate the error-in-the-middle stream.job_manager._job_state = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("status") - + # mocking the response for STATUS CHECKS json_mock_response = request.getfixturevalue(job_response) if job_response else None requests_mock.post(stream.job_manager.base_url, json=json_mock_response) - + # testing raised exception and backoff with pytest.raises(error_type) as error: stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) - + # we expect different call_count, because we set the different max_retries assert expected_msg in repr(error.value) and requests_mock.call_count == call_count_expected @@ -302,11 +311,11 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a job_result_url = test_job_status_response.json().get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + # mocking the nested request call to retrieve the data from result URL stream.job_manager._job_id = job_id requests_mock.get(job_result_url, json=request.getfixturevalue(job_response)) - + # calling the sceario processing stream.job_manager._job_track_running() assert stream.job_manager._job_state == expected @@ -316,13 +325,13 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "running_job_response, canceled_job_response, expected", [ ( - "bulk_job_running_with_object_count_and_url_response", - "bulk_job_canceled_with_object_count_and_url_response", + "bulk_job_running_with_object_count_and_url_response", + "bulk_job_canceled_with_object_count_and_url_response", "bulk-123456789.jsonl", ), ( - "bulk_job_running_with_object_count_no_url_response", - "bulk_job_canceled_with_object_count_no_url_response", + "bulk_job_running_with_object_count_no_url_response", + "bulk_job_canceled_with_object_count_no_url_response", None, ), ], @@ -331,7 +340,9 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "self-canceled with no url", ], ) -def test_job_running_with_canceled_scenario(mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected) -> None: +def test_job_running_with_canceled_scenario( + mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # modify the sleep time for the test stream.job_manager._job_check_interval = 0 @@ -339,7 +350,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_id = request.getfixturevalue(running_job_response).get("data", {}).get("node", {}).get("id") # mocking the response for STATUS CHECKS requests_mock.post( - stream.job_manager.base_url, + stream.job_manager.base_url, [ {"json": request.getfixturevalue(running_job_response)}, {"json": request.getfixturevalue(canceled_job_response)}, @@ -348,7 +359,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_result_url = request.getfixturevalue(canceled_job_response).get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + stream.job_manager._job_id = job_id stream.job_manager._job_checkpoint_interval = 5 # faking self-canceled job @@ -448,9 +459,9 @@ def test_bulk_stream_parse_response( ) def test_stream_slices( auth_config, - stream, - stream_state, - with_start_date, + stream, + stream_state, + with_start_date, expected_start, ) -> None: # simulating `None` for `start_date` and `config migration` @@ -462,7 +473,7 @@ def test_stream_slices( test_result = list(stream.stream_slices(stream_state=stream_state)) assert test_result[0].get("start") == expected_start - + @pytest.mark.parametrize( "stream, json_content_example, last_job_elapsed_time, previous_slice_size, adjusted_slice_size", [ @@ -471,7 +482,7 @@ def test_stream_slices( ids=[ "Expand Slice Size", ], -) +) def test_expand_stream_slices_job_size( request, requests_mock, @@ -483,7 +494,6 @@ def test_expand_stream_slices_job_size( adjusted_slice_size, auth_config, ) -> None: - stream = stream(auth_config) # get the mocked job_result_url test_result_url = bulk_job_completed_response.get("data").get("node").get("url") @@ -495,7 +505,7 @@ def test_expand_stream_slices_job_size( # for the sake of simplicity we fake some parts to simulate the `current_job_time_elapsed` # fake current slice interval value stream.job_manager._job_size = previous_slice_size - # fake `last job elapsed time` + # fake `last job elapsed time` if last_job_elapsed_time: stream.job_manager._job_last_elapsed_time = last_job_elapsed_time diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py index 8a5d7012d38b..cf586b159ae1 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -30,12 +30,12 @@ def test_query_status() -> None: } } }""" - + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.status(input_job_id) assert repr(template) == repr(expected) - - + + def test_bulk_query_prepare() -> None: expected = '''mutation { bulkOperationRunQuery( @@ -54,14 +54,14 @@ def test_bulk_query_prepare() -> None: } } }''' - + input_query_from_slice = "{some_query}" template = ShopifyBulkTemplates.prepare(input_query_from_slice) assert repr(template) == repr(expected) - - + + def test_bulk_query_cancel() -> None: - expected = '''mutation { + expected = """mutation { bulkOperationCancel(id: "gid://shopify/BulkOperation/4047052112061") { bulkOperation { id @@ -73,32 +73,32 @@ def test_bulk_query_cancel() -> None: message } } - }''' - + }""" + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.cancel(input_job_id) assert repr(template) == repr(expected) - + @pytest.mark.parametrize( "query_name, fields, filter_field, start, end, expected", [ ( - "test_root", - ["test_field1", "test_field2"], + "test_root", + ["test_field1", "test_field2"], "updated_at", "2023-01-01", - "2023-01-02", + "2023-01-02", Query( - name='test_root', + name="test_root", arguments=[ - Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=["test_field1", "test_field2"])])] - ) + Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), + ], + fields=[Field(name="edges", fields=[Field(name="node", fields=["test_field1", "test_field2"])])], + ), ) ], - ids=["simple query with filter and sort"] + ids=["simple query with filter and sort"], ) def test_base_build_query(basic_config, query_name, fields, filter_field, start, end, expected) -> None: """ @@ -116,7 +116,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, } ''' """ - + builder = ShopifyBulkQuery(basic_config) filter_query = f"{filter_field}:>'{start}' AND {filter_field}:<='{end}'" built_query = builder.build(query_name, fields, filter_query) @@ -136,14 +136,52 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='customers', + name="customers", arguments=[ Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - Argument(name="sortKey", value="UPDATED_AT"), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=['__typename', 'id', Field(name="updatedAt", alias="customers_updated_at"), Field(name="metafields", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", "namespace", "value", "key", "description", "createdAt", "updatedAt", "type"])])])])])] + Argument(name="sortKey", value="UPDATED_AT"), + ], + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + Field(name="updatedAt", alias="customers_updated_at"), + Field( + name="metafields", + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + "namespace", + "value", + "key", + "description", + "createdAt", + "updatedAt", + "type", + ], + ) + ], + ) + ], + ), + ], + ) + ], + ) + ], ) - ] + ], ), ), ( @@ -206,28 +244,28 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "createdAt", "updatedAt", "type", - ] + ], ) - ] + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] - ) + ], + ), ), ( InventoryLevel, @@ -239,64 +277,91 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='locations', + name="locations", arguments=[ Argument(name="includeLegacy", value="true"), Argument(name="includeInactive", value="true"), - ], + ], fields=[ Field( - name='edges', + name="edges", fields=[ Field( - name='node', + name="node", fields=[ - '__typename', - 'id', + "__typename", + "id", Field( - name="inventoryLevels", + name="inventoryLevels", arguments=[ - Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - ], + Argument( + name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\"" + ), + ], fields=[ Field( - name="edges", + name="edges", fields=[ Field( - name="node", + name="node", fields=[ - "__typename", - "id", - "canDeactivate", - "createdAt", - "deactivationAlert", - "updatedAt", - Field(name="item", fields=[Field(name="inventoryHistoryUrl", alias="inventory_history_url"), Field(name="id", alias="inventory_item_id"), Field(name="locationsCount", alias="locations_count", fields=["count"])]), - Field( - name="quantities", + "__typename", + "id", + "canDeactivate", + "createdAt", + "deactivationAlert", + "updatedAt", + Field( + name="item", + fields=[ + Field( + name="inventoryHistoryUrl", alias="inventory_history_url" + ), + Field(name="id", alias="inventory_item_id"), + Field( + name="locationsCount", + alias="locations_count", + fields=["count"], + ), + ], + ), + Field( + name="quantities", arguments=[ - Argument(name="names", value=['"available"', '"incoming"', '"committed"', '"damaged"', '"on_hand"', '"quality_control"', '"reserved"', '"safety_stock"']) - ], + Argument( + name="names", + value=[ + '"available"', + '"incoming"', + '"committed"', + '"damaged"', + '"on_hand"', + '"quality_control"', + '"reserved"', + '"safety_stock"', + ], + ) + ], fields=[ "id", "name", "quantity", "updatedAt", ], - ) - ] + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] + ], ), ), ], @@ -304,7 +369,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "MetafieldCustomers query with 1 query_path(str)", "MetafieldProductImages query with composite quey_path(List[2])", "InventoryLevel query", - ] + ], ) def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_stream_class, expected) -> None: if parent_stream_class: @@ -314,4 +379,4 @@ def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_s else: stream_query = query_class(auth_config) - assert stream_query.get(filter_field, start, end) == expected.render() \ No newline at end of file + assert stream_query.get(filter_field, start, end) == expected.render() diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py index dff60ea605d5..85c48ce6d13b 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py @@ -58,7 +58,7 @@ def test_check_type(basic_config, record, types, expected) -> None: "alias_to_id_field": "gid://shopify/Metafield/123", "__parentId": "gid://shopify/Order/102030", }, - ) + ), ], ) def test_record_resolver(basic_config, record, expected) -> None: diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py index df16077abc14..d878a1c4e9a4 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py @@ -2,11 +2,13 @@ import json +from source_shopify.scopes import SCOPES_MAPPING +from source_shopify.streams.base_streams import ShopifyStream + from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS from airbyte_cdk.test.mock_http.response_builder import find_template -from source_shopify.scopes import SCOPES_MAPPING -from source_shopify.streams.base_streams import ShopifyStream + _ALL_SCOPES = [scope for stream_scopes in SCOPES_MAPPING.values() for scope in stream_scopes] diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py index 05b15d05dc00..29ae2d9ae93e 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py @@ -4,10 +4,11 @@ from datetime import datetime from random import randint -from airbyte_cdk.test.mock_http import HttpRequest, HttpResponse from source_shopify.shopify_graphql.bulk.query import ShopifyBulkTemplates from source_shopify.streams.base_streams import ShopifyStream +from airbyte_cdk.test.mock_http import HttpRequest, HttpResponse + def _create_job_url(shop_name: str) -> str: return f"https://{shop_name}.myshopify.com/admin/api/{ShopifyStream.api_version}/graphql.json" @@ -42,16 +43,15 @@ def create_job_creation_body(lower_boundary: datetime, upper_boundary: datetime) } } }""" - query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace("%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat()) + query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace( + "%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat() + ) prepared_query = ShopifyBulkTemplates.prepare(query) return json.dumps({"query": prepared_query}) def create_job_creation_request(shop_name: str, lower_boundary: datetime, upper_boundary: datetime) -> HttpRequest: - return HttpRequest( - url=_create_job_url(shop_name), - body=create_job_creation_body(lower_boundary, upper_boundary) - ) + return HttpRequest(url=_create_job_url(shop_name), body=create_job_creation_body(lower_boundary, upper_boundary)) def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: @@ -70,7 +70,7 @@ def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: partialDataUrl }} }} - }}""" + }}""", ) @@ -89,33 +89,26 @@ def create_job_cancel_request(shop_name: str, job_id: str) -> HttpRequest: message }} }} - }}""" + }}""", ) + class JobCreationResponseBuilder: - def __init__(self, job_created_at:str="2024-05-05T02:00:00Z") -> None: + def __init__(self, job_created_at: str = "2024-05-05T02:00:00Z") -> None: self._template = { "data": { "bulkOperationRunQuery": { - "bulkOperation": { - "id": "gid://shopify/BulkOperation/0", - "status": "CREATED", - "createdAt": f"{job_created_at}" - }, - "userErrors": [] + "bulkOperation": {"id": "gid://shopify/BulkOperation/0", "status": "CREATED", "createdAt": f"{job_created_at}"}, + "userErrors": [], } }, "extensions": { "cost": { "requestedQueryCost": 10, "actualQueryCost": 10, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1990, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1990, "restoreRate": 100.0}, } - } + }, } def with_bulk_operation_id(self, bulk_operation_id: str) -> "JobCreationResponseBuilder": @@ -135,30 +128,26 @@ def __init__(self) -> None: "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1999, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1999, "restoreRate": 100.0}, } - } + }, } } - def with_running_status(self, bulk_operation_id: str, object_count: str="10") -> "JobStatusResponseBuilder": + def with_running_status(self, bulk_operation_id: str, object_count: str = "10") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { - "id": bulk_operation_id, - "status": "RUNNING", - "errorCode": None, - "createdAt": "2024-05-28T18:57:54Z", - "objectCount": object_count, - "fileSize": None, - "url": None, - "partialDataUrl": None, + "id": bulk_operation_id, + "status": "RUNNING", + "errorCode": None, + "createdAt": "2024-05-28T18:57:54Z", + "objectCount": object_count, + "fileSize": None, + "url": None, + "partialDataUrl": None, } return self - def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "COMPLETED", @@ -167,11 +156,11 @@ def with_completed_status(self, bulk_operation_id: str, job_result_url: str, obj "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self - def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "CANCELED", @@ -180,7 +169,7 @@ def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, obje "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self @@ -192,15 +181,14 @@ class MetafieldOrdersJobResponseBuilder: def __init__(self) -> None: self._records = [] - def _any_record(self, updated_at:str="2024-05-05T01:09:50Z") -> str: + def _any_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> str: an_id = str(randint(1000000000000, 9999999999999)) a_parent_id = str(randint(1000000000000, 9999999999999)) return f"""{{"__typename":"Order","id":"gid:\/\/shopify\/Order\/{a_parent_id}"}} {{"__typename":"Metafield","id":"gid:\/\/shopify\/Metafield\/{an_id}","namespace":"my_fields","value":"asdfasdf","key":"purchase_order","description":null,"createdAt":"2023-04-13T12:09:50Z","updatedAt":"{updated_at}","type":"single_line_text_field","__parentId":"gid:\/\/shopify\/Order\/{a_parent_id}"}} """ - - def with_record(self, updated_at:str="2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": + def with_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": self._records.append(self._any_record(updated_at=updated_at)) return self diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py index 8a0d6d5c8c59..ad8d1e31fb37 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py @@ -5,17 +5,20 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse + _AN_ERROR_RESPONSE = HttpResponse(json.dumps({"errors": ["an error"]})) _SERVICE_UNAVAILABLE_ERROR_RESPONSE = HttpResponse(json.dumps({"errors": ["Service unavailable"]}), status_code=503) -from airbyte_cdk.models import AirbyteStateMessage, SyncMode -from airbyte_cdk.test.state_builder import StateBuilder from freezegun import freeze_time from requests.exceptions import ConnectionError from source_shopify import SourceShopify + +from airbyte_cdk.models import AirbyteStateMessage, SyncMode +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.integration.api.authentication import grant_all_scopes, set_up_shop from unit_tests.integration.api.bulk import ( JobCreationResponseBuilder, @@ -27,6 +30,7 @@ create_job_status_request, ) + _BULK_OPERATION_ID = "gid://shopify/BulkOperation/4472588009661" _BULK_STREAM = "metafield_orders" _SHOP_NAME = "airbyte-integration-test" @@ -41,6 +45,7 @@ _INCREMENTAL_JOB_START_DATE = datetime.fromisoformat(_INCREMENTAL_JOB_START_DATE_ISO) _INCREMENTAL_JOB_END_DATE = _INCREMENTAL_JOB_START_DATE + timedelta(hours=24, minutes=0) + def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_interval=200000) -> Dict[str, Any]: return { "start_date": start_date.strftime("%Y-%m-%d"), @@ -50,13 +55,12 @@ def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_inter "api_password": "api_password", }, "bulk_window_in_days": bulk_window, - "job_checkpoint_interval": job_checkpoint_interval + "job_checkpoint_interval": job_checkpoint_interval, } @freeze_time(_JOB_END_DATE) class GraphQlBulkStreamTest(TestCase): - def setUp(self) -> None: self._http_mocker = HttpMocker() self._http_mocker.__enter__() @@ -103,8 +107,10 @@ def test_given_response_is_not_json_on_job_creation_when_read_then_retry(self) - job_creation_request, [ HttpResponse("This is not json"), - JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), # This will never get called (see assertion below) - ] + JobCreationResponseBuilder() + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), # This will never get called (see assertion below) + ], ) self._http_mocker.post( @@ -126,8 +132,11 @@ def test_given_connection_error_on_job_creation_when_read_then_retry_job_creatio inner_mocker.register_uri( # TODO the testing library should have the ability to generate ConnectionError. As this might not be trivial, we will wait for another case before implementing "POST", _URL_GRAPHQL, - [{"exc": ConnectionError("ConnectionError")}, {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}], - additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE) + [ + {"exc": ConnectionError("ConnectionError")}, + {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}, + ], + additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -152,7 +161,7 @@ def test_given_retryable_error_on_first_get_job_status_when_read_then_retry(self [ _AN_ERROR_RESPONSE, JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -175,7 +184,7 @@ def test_given_retryable_error_on_get_job_status_when_read_then_retry(self) -> N JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID).build(), HttpResponse(json.dumps({"errors": ["an error"]})), JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -209,7 +218,9 @@ def test_when_read_then_extract_records(self) -> None: job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=5) self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -220,14 +231,10 @@ def test_when_read_then_extract_records(self) -> None: MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), ) # expectation is job start date should be the updated_at in orders - metafield_orders_orders_state = {"orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + metafield_orders_orders_state = { + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago @@ -238,13 +245,13 @@ def test_when_read_then_extract_records(self) -> None: assert len(output.records) == 2 def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(self) -> None: - """" + """ " The motivation of this test is https://github.com/airbytehq/oncall/issues/6874 - + In this scenario we end having stream_slices method to generate same slice N times. Our checkpointing logic will trigger when job_checkpoint_interval is passed, but there may be the case that such checkpoint has the same value as the current slice start date so we would end requesting same job. - + In this test: 1. First job requires to checkpoint as we pass the 1500 limit, it cancels the bulk job and checkpoints from last cursor value. 2. Next job just goes "fine". @@ -261,6 +268,7 @@ def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(s {"type":"LOG","log":{"level":"INFO","message":"Stream: `metafield_orders`, the BULK Job: `gid://shopify/BulkOperation/4472588009771` is CREATED"}} ... """ + def add_n_records(builder, n, record_date: Optional[str] = None): for _ in range(n): builder = builder.with_record(updated_at=record_date) @@ -271,7 +279,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # create a job request self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) # get job status self._http_mocker.post( @@ -282,15 +292,10 @@ def add_n_records(builder, n, record_date: Optional[str] = None): JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID, object_count="16000").build(), # this will complete the job JobStatusResponseBuilder().with_canceled_status(_BULK_OPERATION_ID, _JOB_RESULT_URL, object_count="1700").build(), - ] + ], ) # mock the cancel operation request as we passed the 15000 rows - self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), - [ - HttpResponse(json.dumps({}), status_code=200) - ] - ) + self._http_mocker.post(create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), [HttpResponse(json.dumps({}), status_code=200)]) # get results for the request that got cancelled adjusted_checkpoint_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(days=2, hours=6, minutes=30) adjusted_record_date = adjusted_checkpoint_start_date.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -309,7 +314,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): self._http_mocker.post( # The start date is caused by record date in previous iteration create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693950.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693950.jsonl&response-content-type=application%2Fjsonl" @@ -318,7 +325,7 @@ def add_n_records(builder, n, record_date: Optional[str] = None): [ # this will output the job is running JobStatusResponseBuilder().with_completed_status(next_bulk_operation_id, next_job_result_url).build(), - ] + ], ) # get results for the request that got cancelled self._http_mocker.get( @@ -333,12 +340,14 @@ def add_n_records(builder, n, record_date: Optional[str] = None): adjusted_checkpoint_end_date = adjusted_checkpoint_start_date + timedelta(days=1) job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=4) create_job_request = create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date) - + self._http_mocker.post( create_job_request, - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) - + base_status_responses = [ JobStatusResponseBuilder().with_running_status(next_bulk_operation_id, object_count="500").build(), # this should make the job get canceled as it gets over 15000 rows @@ -346,23 +355,17 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # this will complete the job JobStatusResponseBuilder().with_canceled_status(next_bulk_operation_id, next_job_result_url, object_count="1700").build(), ] - + n_times_to_loop = 4 responses_in_loop = base_status_responses * n_times_to_loop # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693960.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693960.jsonl&response-content-type=application%2Fjsonl" - - self._http_mocker.post( - create_job_status_request(_SHOP_NAME, next_bulk_operation_id), - responses_in_loop - ) + + self._http_mocker.post(create_job_status_request(_SHOP_NAME, next_bulk_operation_id), responses_in_loop) # mock the cancel operation request as we passed the 15000 rows self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), - [ - HttpResponse(json.dumps({}), status_code=200) - ] + create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), [HttpResponse(json.dumps({}), status_code=200)] ) # get results @@ -371,35 +374,28 @@ def add_n_records(builder, n, record_date: Optional[str] = None): HttpRequest(next_job_result_url), add_n_records(MetafieldOrdersJobResponseBuilder(), 80, adjusted_record_date).build(), ) - + # ********* end of request mocking ************* metafield_orders_orders_state = { - "orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago config_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(weeks=104) - output = self._read(_get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state) + output = self._read( + _get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state + ) expected_error_message = "The stream: `metafield_orders` checkpoint collision is detected." result = output.errors[0].trace.error.internal_message - # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` + # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` assert result is not None and expected_error_message in result - def _read(self, config, sync_mode=SyncMode.full_refresh, state: Optional[List[AirbyteStateMessage]] = None): catalog = CatalogBuilder().with_stream(_BULK_STREAM, sync_mode).build() output = read(SourceShopify(), config, catalog, state=state) return output - - - diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py index ee1ae56a74a1..932fc7a33fa7 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py @@ -7,6 +7,7 @@ from source_shopify.auth import NotImplementedAuth, ShopifyAuthenticator from source_shopify.source import ConnectionCheckTest + TEST_ACCESS_TOKEN = "test_access_token" TEST_API_PASSWORD = "test_api_password" diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py index 80cc6115c449..718ebb2537c2 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py @@ -7,6 +7,7 @@ from source_shopify.streams.streams import OrderRefunds, Orders from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache + # Define the Stream instances for the tests SHOPIFY_STREAM = Orders(config={"authenticator": None}) SHOPIFY_SUB_STREAM = OrderRefunds(config={"authenticator": None}) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py index 49850a1ef8a1..daedb2a89a9d 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py @@ -6,6 +6,7 @@ import requests from source_shopify.utils import ShopifyRateLimiter as limiter + TEST_DATA_FIELD = "some_data_field" TEST_RATE_LIMIT_HEADER = "X-Shopify-Shop-Api-Call-Limit" TEST_THRESHOLD = 0.9 diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py index 3d599ce770b1..ea902f0be567 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py @@ -108,7 +108,9 @@ def test_read_deleted_records(stream, requests_mock, deleted_records_json, expec stream = stream(config) deleted_records_url = stream.url_base + stream.deleted_events.path() requests_mock.get(deleted_records_url, json=deleted_records_json) - mocker.patch("source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json) + mocker.patch( + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json + ) assert list(stream.read_records(sync_mode=None)) == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py index a6c99f9c3c44..285b816734ed 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py @@ -7,10 +7,25 @@ @pytest.mark.parametrize( "page_size, filter_value, next_page_token, expected_query", [ - (100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + ( + 100, + None, + None, + "query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 200, + "2027-07-11T13:07:45-07:00", + None, + "query {\n products(first: 200, query: \"updated_at:>'2027-07-11T13:07:45-07:00'\", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 250, + "2027-07-11T13:07:45-07:00", + "end_cursor_value", + 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}', + ), ], ) def test_get_query_products(page_size, filter_value, next_page_token, expected_query): - assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query + assert get_query_products(page_size, "updatedAt", filter_value, next_page_token) == expected_query diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py index de54e242294a..c90b81052dda 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_shopify.config_migrations import MigrateConfig from source_shopify.source import SourceShopify +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index f9ef2ba2f7ca..797be1e385f4 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_shopify.auth import ShopifyAuthenticator from source_shopify.source import ConnectionCheckTest, SourceShopify from source_shopify.streams.streams import ( @@ -49,6 +48,8 @@ TransactionsGraphql, ) +from airbyte_cdk.utils import AirbyteTracedException + from .conftest import records_per_slice @@ -126,13 +127,13 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi else: result = stream.path() assert result == expected_path - - + + @pytest.mark.parametrize( "stream, parent_records, state_checkpoint_interval", [ ( - OrderRefunds, + OrderRefunds, [ {"id": 1, "refunds": [{"created_at": "2021-01-01T00:00:00+00:00"}]}, {"id": 2, "refunds": [{"created_at": "2021-02-01T00:00:00+00:00"}]}, @@ -145,10 +146,10 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi ], ) def test_stream_slice_nested_substream_buffering( - mocker, - config, - stream, - parent_records, + mocker, + config, + stream, + parent_records, state_checkpoint_interval, ) -> None: # making the stream instance @@ -156,7 +157,7 @@ def test_stream_slice_nested_substream_buffering( stream.state_checkpoint_interval = state_checkpoint_interval # simulating `read_records` for the `parent_stream` mocker.patch( - "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=parent_records, ) # count how many slices we expect, based on the number of parent_records @@ -173,7 +174,7 @@ def test_stream_slice_nested_substream_buffering( # count total slices total_slices += 1 # check we have emitted complete number of slices - assert total_slices == total_slices_expected + assert total_slices == total_slices_expected def test_check_connection(config, mocker) -> None: @@ -212,11 +213,23 @@ def test_request_params(config, stream, expected) -> None: "last_record, current_state, expected", [ # no init state - ({"created_at": "2022-10-10T06:21:53-07:00"}, {}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is empty str - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": ""}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": ""}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is None - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": None}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": None}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # last rec cursor is None ({"created_at": None}, {"created_at": None}, {"created_at": "", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), # last rec cursor is empty str @@ -257,21 +270,19 @@ def test_get_shop_name(config, shop, expected) -> None: actual = source.get_shop_name(config) assert actual == expected + @pytest.mark.parametrize( "config, expected_stream_class", [ ({"fetch_transactions_user_id": False}, TransactionsGraphql), ({"fetch_transactions_user_id": True}, Transactions), ({}, TransactionsGraphql), - ], + ], ids=["don't fetch user_id", "fetch user id", "unset config value shouldn't fetch user_id"], ) def test_select_transactions_stream(config, expected_stream_class): config["shop"] = "test-store" - config["credentials"] = { - "auth_method": "api_password", - "api_password": "shppa_123" - } + config["credentials"] = {"auth_method": "api_password", "api_password": "shppa_123"} config["authenticator"] = ShopifyAuthenticator(config) source = SourceShopify() @@ -284,7 +295,7 @@ def test_select_transactions_stream(config, expected_stream_class): [ pytest.param([{"id": "12345"}], "12345", None, id="test_shop_name_exists"), pytest.param([], None, AirbyteTracedException, id="test_shop_name_does_not_exist"), - ], + ], ) def test_get_shop_id(config, read_records, expected_shop_id, expected_error): check_test = ConnectionCheckTest(config) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index ca9e306e6698..649e409e7056 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -100,8 +100,9 @@ def test_privileges_validation(requests_mock, fetch_transactions_user_id, basic_ "Internal Server Error for slice (500)", ], ) -def test_unavailable_stream(requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, - json_response: Mapping[str, Any]): +def test_unavailable_stream( + requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, json_response: Mapping[str, Any] +): stream = stream(auth_config) url = stream.url_base + stream.path(stream_slice=slice) requests_mock.get(url=url, json=json_response, status_code=status_code) diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py index d49b55882333..a9256a533972 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-slack/main.py b/airbyte-integrations/connectors/source-slack/main.py index b2ff9c851163..f80fc7bb2ca1 100644 --- a/airbyte-integrations/connectors/source-slack/main.py +++ b/airbyte-integrations/connectors/source-slack/main.py @@ -4,5 +4,6 @@ from source_slack.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py index 9dbb401a07e9..0ee57c4d76ad 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py @@ -4,6 +4,7 @@ from typing import List import requests + from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py index 02bd7ee30866..ec353d4921a1 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py @@ -5,6 +5,7 @@ from typing import Any, Iterable, List, Mapping, Optional import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever @@ -13,6 +14,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + LOGGER = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py b/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py index 0db32e4bce13..cba204c928bd 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py @@ -4,9 +4,10 @@ import logging from typing import Optional, Union -from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy from requests import RequestException, Response +from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy + class SlackBackoffStrategy(BackoffStrategy): def __init__(self, logger: logging.Logger): diff --git a/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py index cc6d9cd03607..aabfeadd3c7e 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from source_slack import SourceSlack + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 5328556a1624..bf600c7a6bc9 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping import pendulum + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-slack/source_slack/streams.py b/airbyte-integrations/connectors/source-slack/source_slack/streams.py index 1530c74a0ae5..8623af137fef 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/streams.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/streams.py @@ -8,12 +8,13 @@ import pendulum import requests +from pendulum import DateTime + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler, HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING -from pendulum import DateTime from source_slack.components.slack_backoff_strategy import SlackBackoffStrategy from .components.join_channels import JoinChannelsStream diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 002a9ec96779..3d03f4755646 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -8,6 +8,7 @@ import pytest + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" @@ -16,13 +17,10 @@ def conversations_list(requests_mock): return requests_mock.register_uri( "GET", "https://slack.com/api/conversations.list?limit=1000&types=public_channel", - json={ - "channels": [ - {"id": "airbyte-for-beginners", "is_member": True}, - {"id": "good-reads", "is_member": True}] - }, + json={"channels": [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}]}, ) + def base_config() -> MutableMapping: return copy.deepcopy( { @@ -98,11 +96,27 @@ def invalid_config() -> MutableMapping: @pytest.fixture def joined_channel(): - return {"id": "C061EG9SL", "name": "general", "is_channel": True, "is_group": False, "is_im": False, - "created": 1449252889, - "creator": "U061F7AUR", "is_archived": False, "is_general": True, "unlinked": 0, "name_normalized": "general", - "is_shared": False, - "is_ext_shared": False, "is_org_shared": False, "pending_shared": [], "is_pending_ext_shared": False, - "is_member": True, "is_private": False, "is_mpim": False, - "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, - "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, "previous_names": []} + return { + "id": "C061EG9SL", + "name": "general", + "is_channel": True, + "is_group": False, + "is_im": False, + "created": 1449252889, + "creator": "U061F7AUR", + "is_archived": False, + "is_general": True, + "unlinked": 0, + "name_normalized": "general", + "is_shared": False, + "is_ext_shared": False, + "is_org_shared": False, + "pending_shared": [], + "is_pending_ext_shared": False, + "is_member": True, + "is_private": False, + "is_mpim": False, + "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, + "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, + "previous_names": [], + } diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py index dc21220a0139..d0068d1f4aa9 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py @@ -4,13 +4,14 @@ import pendulum import pytest +from source_slack import SourceSlack +from source_slack.components.channel_members_extractor import ChannelMembersExtractor +from source_slack.components.join_channels import ChannelsRetriever, JoinChannelsStream + from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordSelector from airbyte_cdk.sources.declarative.requesters import HttpRequester from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_protocol.models import SyncMode -from source_slack import SourceSlack -from source_slack.components.channel_members_extractor import ChannelMembersExtractor -from source_slack.components.join_channels import ChannelsRetriever, JoinChannelsStream def get_stream_by_name(stream_name, config): @@ -23,22 +24,13 @@ def get_stream_by_name(stream_name, config): def test_channel_members_extractor(token_config): response_mock = MagicMock() - response_mock.json.return_value = {"members": [ - "U023BECGF", - "U061F7AUR", - "W012A3CDE" - ]} + response_mock.json.return_value = {"members": ["U023BECGF", "U061F7AUR", "W012A3CDE"]} records = ChannelMembersExtractor(config=token_config, parameters={}, field_path=["members"]).extract_records(response=response_mock) - assert records == [{"member_id": "U023BECGF"}, - {"member_id": "U061F7AUR"}, - {"member_id": "W012A3CDE"}] + assert records == [{"member_id": "U023BECGF"}, {"member_id": "U061F7AUR"}, {"member_id": "W012A3CDE"}] def test_join_channels(token_config, requests_mock, joined_channel): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json={"ok": True, "channel": joined_channel} - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json={"ok": True, "channel": joined_channel}) token = token_config["credentials"]["api_token"] authenticator = TokenAuthenticator(token) channel_filter = token_config["channel_filter"] @@ -51,13 +43,16 @@ def test_join_channels(token_config, requests_mock, joined_channel): def get_channels_retriever_instance(token_config): return ChannelsRetriever( config=token_config, - requester=HttpRequester(name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, - parameters={}), + requester=HttpRequester( + name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, parameters={} + ), record_selector=RecordSelector( extractor=DpathExtractor(field_path=["channels"], config=token_config, parameters={}), - config=token_config, parameters={}, - schema_normalization=None), - parameters={} + config=token_config, + parameters={}, + schema_normalization=None, + ), + parameters={}, ) @@ -76,20 +71,22 @@ def test_join_channels_make_join_channel_slice(token_config): @pytest.mark.parametrize( "join_response, log_message", ( - ({"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, "Successfully joined channel: test channel"), - ({"ok": False, "error": "missing_scope", "needed": "channels:write"}, - "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}"), + ( + {"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, + "Successfully joined channel: test channel", + ), + ( + {"ok": False, "error": "missing_scope", "needed": "channels:write"}, + "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}", + ), ), - ids=["successful_join_to_channel", "failed_join_to_channel"] + ids=["successful_join_to_channel", "failed_join_to_channel"], ) def test_join_channel_read(requests_mock, token_config, joined_channel, caplog, join_response, log_message): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json=join_response - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json=join_response) requests_mock.get( url="https://slack.com/api/conversations.list", - json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]} + json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]}, ) retriever = get_channels_retriever_instance(token_config) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py index 761597a66fc2..7ceae8c61119 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py @@ -7,6 +7,7 @@ from source_slack import SourceSlack from source_slack.config_migrations import MigrateLegacyConfig + CMD = "check" TEST_CONFIG_LEGACY_PATH = f"{os.path.dirname(__file__)}/configs/legacy_config.json" TEST_CONFIG_ACTUAL_PATH = f"{os.path.dirname(__file__)}/configs/actual_config.json" diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index ae1a58922797..90f6531ca258 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -17,6 +17,7 @@ def get_stream_by_name(stream_name, config): return stream raise ValueError(f"Stream {stream_name} not found") + @parametrized_configs def test_streams(conversations_list, config, is_valid): source = SourceSlack() diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index b9966eb3594c..4ec3b24c517a 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -6,12 +6,13 @@ import pendulum import pytest -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from requests import Response from source_slack import SourceSlack from source_slack.streams import Channels, JoinChannelsStream, Threads +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.fixture def authenticator(token_config): @@ -36,10 +37,10 @@ def get_stream_by_name(stream_name, config): {}, [ # two messages per each channel - {'channel': 'airbyte-for-beginners', 'ts': 1577866844}, - {'channel': 'airbyte-for-beginners', 'ts': 1577877406}, - {'channel': 'good-reads', 'ts': 1577866844}, - {'channel': 'good-reads', 'ts': 1577877406}, + {"channel": "airbyte-for-beginners", "ts": 1577866844}, + {"channel": "airbyte-for-beginners", "ts": 1577877406}, + {"channel": "good-reads", "ts": 1577866844}, + {"channel": "good-reads", "ts": 1577877406}, ], ), ("2020-01-02T00:00:00Z", "2020-01-01T00:00:00Z", [], {}, [{}]), @@ -55,18 +56,18 @@ def get_stream_by_name(stream_name, config): ), ), ) -def test_threads_stream_slices( - requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result -): +def test_threads_stream_slices(requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result): token_config["channel_filter"] = [] requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) start_date = pendulum.parse(start_date) @@ -76,7 +77,7 @@ def test_threads_stream_slices( authenticator=authenticator, default_start_date=start_date, end_date=end_date, - lookback_window=pendulum.Duration(days=token_config["lookback_window"]) + lookback_window=pendulum.Duration(days=token_config["lookback_window"]), ) slices = list(stream.stream_slices(stream_state=stream_state)) assert slices == expected_result @@ -92,11 +93,10 @@ def test_threads_stream_slices( ), ) def test_get_updated_state(authenticator, token_config, current_state, latest_record, expected_state): - stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) assert stream._get_updated_state(current_stream_state=current_state, latest_record=latest_record) == expected_state @@ -105,10 +105,10 @@ def test_threads_request_params(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - expected = {'channel': 'airbyte-for-beginners', 'limit': 1000, 'ts': 1577866844} + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + expected = {"channel": "airbyte-for-beginners", "limit": 1000, "ts": 1577866844} assert stream.request_params(stream_slice=threads_slice, stream_state={}) == expected @@ -116,7 +116,7 @@ def test_threads_parse_response(mocker, authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) resp = { "messages": [ @@ -129,14 +129,14 @@ def test_threads_parse_response(mocker, authenticator, token_config): "subscribed": True, "last_read": "1484678597.521003", "unread_count": 0, - "ts": "1482960137.003543" + "ts": "1482960137.003543", } ] } resp_mock = mocker.Mock() resp_mock.json.return_value = resp - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - actual_response = list(stream.parse_response(response=resp_mock,stream_slice=threads_slice)) + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + actual_response = list(stream.parse_response(response=resp_mock, stream_slice=threads_slice)) assert len(actual_response) == 1 assert actual_response[0]["float_ts"] == 1482960137.003543 assert actual_response[0]["channel_id"] == "airbyte-for-beginners" @@ -147,7 +147,7 @@ def test_backoff(token_config, authenticator, headers, expected_result): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, headers=headers) assert stream.get_backoff_strategy().backoff_time(mocked_response) == expected_result @@ -157,10 +157,7 @@ def test_channels_stream_with_autojoin(authenticator) -> None: """ The test uses the `conversations_list` fixture(autouse=true) as API mocker. """ - expected = [ - {'id': 'airbyte-for-beginners', 'is_member': True}, - {'id': 'good-reads', 'is_member': True} - ] + expected = [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}] stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) assert list(stream.read_records(None)) == expected @@ -169,7 +166,7 @@ def test_next_page_token(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = Mock() mocked_response.json.return_value = {"response_metadata": {"next_cursor": "next page"}} @@ -189,22 +186,24 @@ def test_should_retry(authenticator, token_config, status_code, expected): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, status_code=status_code) mocked_response.ok = status_code == 200 assert stream.get_error_handler().interpret_response(mocked_response).response_action == expected + def test_channels_stream_with_include_private_channels_false(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=False, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel' + assert params.get("types") == "public_channel" + def test_channels_stream_with_include_private_channels(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=True, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel,private_channel' + assert params.get("types") == "public_channel,private_channel" diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartsheets/main.py b/airbyte-integrations/connectors/source-smartsheets/main.py index 62f5650b92ea..deabf33cba73 100644 --- a/airbyte-integrations/connectors/source-smartsheets/main.py +++ b/airbyte-integrations/connectors/source-smartsheets/main.py @@ -4,5 +4,6 @@ from source_smartsheets.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py index ac0c398907e0..f21e72409f4a 100644 --- a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple import smartsheet + from airbyte_cdk.sources.streams.http.requests_native_auth import SingleUseRefreshTokenOauth2Authenticator diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py index 54e3bb226ee5..285e1da0e19b 100644 --- a/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pytest from smartsheet.models import Sheet + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py index 0e9fd35ce43d..498fd9fb8192 100644 --- a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py @@ -5,9 +5,10 @@ import datetime from unittest.mock import Mock -from airbyte_cdk.models import SyncMode from source_smartsheets.streams import SmartsheetStream +from airbyte_cdk.models import SyncMode + def test_state_saved_after_each_record(config, get_sheet_mocker): today_dt = datetime.datetime.now(datetime.timezone.utc) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index 971f33a69dd1..e81b1f8f0da5 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -5,5 +5,6 @@ from source_stripe.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py index 7365b5902b93..c909371d1ede 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution from source_stripe.error_handlers.stripe_error_handler import StripeErrorHandler diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py index a9fc2f3e9307..315b03c9dbde 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py @@ -6,11 +6,13 @@ from typing import Optional, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + STRIPE_ERROR_CODES = { "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " "Try to grant required permissions/scopes or re-authenticate", diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py index 732ac748556d..70cb8d1d425e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py @@ -5,6 +5,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 404: ErrorResolution( response_action=ResponseAction.IGNORE, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py index 37263eabb998..e6878a153f19 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -8,9 +8,10 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_stripe import SourceStripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 5ce96c24cc82..7002a36f5a4a 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum + from airbyte_cdk.entrypoint import logger as entrypoint_logger from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource @@ -36,6 +37,7 @@ UpdatedCursorIncrementalStripeSubStream, ) + logger = logging.getLogger("airbyte") _MAX_CONCURRENCY = 20 @@ -46,7 +48,6 @@ class SourceStripe(ConcurrentSourceAdapter): - message_repository = InMemoryMessageRepository(entrypoint_logger.level) _SLICE_BOUNDARY_FIELDS_BY_IMPLEMENTATION = { Events: ("created[gte]", "created[lte]"), @@ -583,7 +584,10 @@ def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: ), StripeSubStream( name="usage_records", - path=lambda self, stream_slice, *args, **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", + path=lambda self, + stream_slice, + *args, + **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", parent=subscription_items, primary_key=None, **args, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 5eec772a4f85..3f1ca33aefea 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -11,6 +11,7 @@ import pendulum import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ExponentialBackoffStrategy @@ -24,6 +25,7 @@ from source_stripe.error_handlers import ParentIncrementalStripeSubStreamErrorHandler, StripeErrorHandler from source_stripe.error_mappings import PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING + STRIPE_API_VERSION = "2022-11-15" CACHE_DISABLED = os.environ.get("CACHE_DISABLED") IS_TESTING = os.environ.get("DEPLOYMENT_MODE") == "testing" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index f0bcd0cb9e71..da3a15fd2e0f 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -5,10 +5,12 @@ import os import pytest + from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.test.state_builder import StateBuilder + os.environ["CACHE_DISABLED"] = "true" os.environ["DEPLOYMENT_MODE"] = "testing" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py index 35c72a193874..ce2e72a67d48 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py @@ -4,6 +4,8 @@ from unittest import TestCase import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -20,7 +22,7 @@ from integration.config import ConfigBuilder from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder -from source_stripe import SourceStripe + _STREAM_NAME = "accounts" _ACCOUNT_ID = "acct_1G9HZLIEn49ers" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py index 9e8ca1c2628c..dd7a36ebd56a 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, AirbyteStateMessage, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["application_fee.created", "application_fee.refunded"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py index 1e8b500ea988..8581688de825 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py @@ -8,6 +8,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -29,7 +31,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["application_fee.refund.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py index cdb0da90e26a..c4a29d2d42d6 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py index bafb8da0bb45..92abc3a33af2 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py @@ -8,6 +8,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -29,7 +31,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py index 660564ea9620..d1c037ea9e65 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_card.created", "issuing_card.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py index b64ba61d6db8..7e0c41a080f4 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["radar.early_fraud_warning.created", "radar.early_fraud_warning.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py index 87c5494b755e..4215ead25260 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _STREAM_NAME = "events" _NOW = datetime.now(timezone.utc) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py index c9f7142af059..95ae64e494a7 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py index 918cab91a40d..3e4f6c2adfa7 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py index f41f2efbbbf5..1c9bb3a5e2af 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -28,7 +30,7 @@ from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status from integration.test_bank_accounts import _a_customer, _customers_response -from source_stripe import SourceStripe + _EVENT_TYPES = ["payment_method.*"] @@ -105,12 +107,7 @@ class FullRefreshTest(TestCase): def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -125,12 +122,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -152,12 +144,7 @@ def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMo def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -172,12 +159,7 @@ def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -190,12 +172,7 @@ def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mock def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -208,12 +185,7 @@ def test_given_http_status_401_when_read_then_config_error(self, http_mocker: Ht def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -229,12 +201,7 @@ def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_m def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -247,12 +214,7 @@ def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_r def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -272,12 +234,7 @@ class IncrementalTest(TestCase): def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) cursor_value = int(_A_START_DATE.timestamp()) + 1 http_mocker.get( diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py index f1a96ed48693..58a3b355ebd6 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py @@ -4,6 +4,8 @@ from unittest import TestCase import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -21,7 +23,7 @@ from integration.config import ConfigBuilder from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder -from source_stripe import SourceStripe + _STREAM_NAME = "payout_balance_transactions" _A_PAYOUT_ID = "a_payout_id" @@ -118,7 +120,10 @@ def test_given_multiple_parents_when_read_then_extract_from_all_children(self, h config = _config().with_start_date(_START_DATE).build() http_mocker.get( _payouts_request().with_created_gte(_START_DATE).with_created_lte(_NOW).with_limit(100).build(), - _payouts_response().with_record(_create_payout_record().with_id(_A_PAYOUT_ID)).with_record(_create_payout_record().with_id(_ANOTHER_PAYOUT_ID)).build(), + _payouts_response() + .with_record(_create_payout_record().with_id(_A_PAYOUT_ID)) + .with_record(_create_payout_record().with_id(_ANOTHER_PAYOUT_ID)) + .build(), ) http_mocker.get( _balance_transactions_request().with_limit(100).with_payout(_A_PAYOUT_ID).build(), @@ -160,8 +165,15 @@ def test_when_read_then_fetch_from_updated_payouts(self, http_mocker: HttpMocker state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(_STATE_DATE.timestamp())}).build() catalog = _create_catalog(SyncMode.incremental) http_mocker.get( - _events_request().with_created_gte(_STATE_DATE + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), - _events_response().with_record(_event_record().with_field(_DATA_FIELD, _create_payout_record().with_id(_A_PAYOUT_ID).build())).build(), + _events_request() + .with_created_gte(_STATE_DATE + _AVOIDING_INCLUSIVE_BOUNDARIES) + .with_created_lte(_NOW) + .with_limit(100) + .with_types(_EVENT_TYPES) + .build(), + _events_response() + .with_record(_event_record().with_field(_DATA_FIELD, _create_payout_record().with_id(_A_PAYOUT_ID).build())) + .build(), ) http_mocker.get( _balance_transactions_request().with_limit(100).with_payout(_A_PAYOUT_ID).build(), diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py index 7779b39bcd03..de4dd3d3126b 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, AirbyteStreamStatus, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _STREAM_NAME = "persons" _ACCOUNT_ID = "acct_1G9HZLIEn49ers" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py index 13da98982181..7bc0c7f3d329 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["review.closed", "review.opened"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py index cf5b850124d0..280df5218352 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_transaction.created", "issuing_transaction.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 1567a930ebbe..98128124534e 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -6,12 +6,14 @@ from contextlib import nullcontext as does_not_raise import pytest +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer, SyncMode from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils import AirbyteTracedException -from source_stripe import SourceStripe + logger = logging.getLogger("airbyte") _ANY_CATALOG = ConfiguredAirbyteCatalogSerializer.load({"streams": []}) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index eaaafcad12ef..666dcb9e961f 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -133,7 +133,6 @@ def test_lazy_substream_data_cursor_value_is_populated( def test_lazy_substream_data_is_expanded( requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state ): - config["start_date"] = str(pendulum.today().subtract(days=3)) stream = stream_by_name("bank_accounts", config) for url, body in requests_mock_map.items(): @@ -523,6 +522,7 @@ def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by ] assert records == [{"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1691629292}] + @freezegun.freeze_time("2023-08-23T15:00:15Z") def test_setup_attempts(requests_mock, incremental_stream_args): requests_mock.get( @@ -583,12 +583,12 @@ def test_setup_attempts(requests_mock, incremental_stream_args): def test_persons_wo_state(requests_mock, stream_args): requests_mock.get("/v1/accounts", json={"data": [{"id": 1, "object": "account", "created": 111}]}) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("full_refresh")) assert slices == [{"parent": {"id": 1, "object": "account", "created": 111}}] requests_mock.get("/v1/accounts/1/persons", json={"data": [{"id": 11, "object": "person", "created": 222}]}) @@ -618,12 +618,12 @@ def test_persons_w_state(requests_mock, stream_args): }, ) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("incremental", stream_state={"updated": pendulum.parse("2023-08-20T00:00:00").int_timestamp})) assert slices == [{}] records = [ @@ -802,7 +802,7 @@ def test_subscription_items_extra_request_params(requests_mock, stream_by_name, "created": 1699603175, "quantity": 1, "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", - "subscription_updated": 1699603174, #1699603175 + "subscription_updated": 1699603174, # 1699603175 }, { "id": "si_OynPdzMZykmCWm", diff --git a/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveycto/main.py b/airbyte-integrations/connectors/source-surveycto/main.py index 9f282dbc2ecd..7bd10098aad5 100644 --- a/airbyte-integrations/connectors/source-surveycto/main.py +++ b/airbyte-integrations/connectors/source-surveycto/main.py @@ -4,5 +4,6 @@ from source_surveycto.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py index 816b0dad755e..f0ff2adbb409 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream @@ -111,7 +112,6 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: # Source class SourceSurveycto(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, Any]: - form_ids = config["form_id"] try: diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 77ebfa14f624..863aa11f2b76 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -26,9 +26,11 @@ def source_fixture(): @pytest.fixture(name="mock_survey_cto") def mock_survey_cto_fixture(): - with patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, patch( - "source_surveycto.source.Helpers.get_filter_data", return_value="value" - ) as mock_filter_data, patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema: + with ( + patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, + patch("source_surveycto.source.Helpers.get_filter_data", return_value="value") as mock_filter_data, + patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema, + ): yield mock_call_survey_cto, mock_filter_data, mock_json_schema diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveymonkey/main.py b/airbyte-integrations/connectors/source-surveymonkey/main.py index bf4f900ad377..4daa260e708e 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/main.py +++ b/airbyte-integrations/connectors/source-surveymonkey/main.py @@ -4,5 +4,6 @@ from source_surveymonkey.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py index 98709c3e5f93..8c96e04e6453 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py index 18dcacd06ea2..f2951a5f1903 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py @@ -7,12 +7,14 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from .streams import Surveys + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 33f1740a68e4..ff46c119e1ef 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -10,11 +10,13 @@ import pendulum import requests import vcr + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream + cache_file = tempfile.NamedTemporaryFile() diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py index 74c9ae98ad01..0d1451270f99 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py @@ -6,37 +6,37 @@ import pendulum import pytest +from source_surveymonkey.source import SourceSurveymonkey + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice -from source_surveymonkey.source import SourceSurveymonkey -@pytest.fixture(name='read_json') +@pytest.fixture(name="read_json") def read_json_fixture(request): def read_json(file_name, skip_folder=False): if not skip_folder: - folder_name = request.node.fspath.basename.split('.')[0] + folder_name = request.node.fspath.basename.split(".")[0] with open("unit_tests/" + folder_name + "/" + file_name) as f: return json.load(f) + return read_json -@pytest.fixture(name='read_records') + +@pytest.fixture(name="read_records") def read_records_fixture(config): def read_records(stream_name, slice=StreamSlice(partition={"survey_id": "307785415"}, cursor_slice={})): stream = next(filter(lambda x: x.name == stream_name, SourceSurveymonkey().streams(config=config))) - records = list( - map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) + records = list(map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) return records + return read_records @pytest.fixture def args_mock(): - return { - "authenticator": None, - "start_date": pendulum.parse("2000-01-01"), - "survey_ids": [] - } + return {"authenticator": None, "start_date": pendulum.parse("2000-01-01"), "survey_ids": []} + @pytest.fixture def config(args_mock): @@ -44,5 +44,5 @@ def config(args_mock): **args_mock, "survey_ids": ["307785415"], "credentials": {"access_token": "access_token"}, - "start_date": args_mock["start_date"].to_iso8601_string() + "start_date": args_mock["start_date"].to_iso8601_string(), } diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py index 4e4fde8edf32..8f42734f7fd0 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py @@ -8,6 +8,7 @@ from source_surveymonkey.config_migrations import MigrateAccessTokenToCredentials from source_surveymonkey.source import SourceSurveymonkey + TEST_CONFIG = "test_old_config.json" NEW_TEST_CONFIG = "test_new_config.json" UPGRADED_TEST_CONFIG = "test_upgraded_config.json" diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py index eaa7b4d15176..5af6707b9a7b 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py @@ -6,9 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from source_surveymonkey.components import SurveyIdPartitionRouter +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig + + # test cases as a list of tuples (survey_ids, parent_stream_configs, expected_slices) test_cases = [ ( diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py index 32ef91a401c6..9800cc794fd7 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py @@ -4,9 +4,10 @@ import pendulum import pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_surveymonkey.streams import Surveys +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + class TestSurveymonkeySource: @staticmethod diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py index a397163a108b..69850063503b 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py @@ -4,6 +4,7 @@ from source_surveymonkey.source import SourceSurveymonkey + source_config = {"start_date": "2021-01-01T00:00:00", "access_token": "something"} new_source_config = { "start_date": "2021-01-01T00:00:00", diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py index 24a60bc08d95..66a14003d899 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -5,14 +5,15 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import SyncMode from source_surveymonkey.streams import SurveyIds, Surveys +from airbyte_cdk.models import SyncMode + -@pytest.mark.parametrize("stream, expected_records_file, stream_slice", [ - (SurveyIds, "records_survey_ids.json", None), - (Surveys, "records_surveys.json", {"survey_id": "307785415"}) -]) +@pytest.mark.parametrize( + "stream, expected_records_file, stream_slice", + [(SurveyIds, "records_survey_ids.json", None), (Surveys, "records_surveys.json", {"survey_id": "307785415"})], +) def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, expected_records_file, stream_slice): requests_mock.get( "https://api.surveymonkey.com/v3/surveys", @@ -30,8 +31,11 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, } }, }, - {"status_code": 200, "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, - "json": read_json("response_survey_ids.json")}, + { + "status_code": 200, + "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, + "json": read_json("response_survey_ids.json"), + }, ], ) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) @@ -42,10 +46,10 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, assert list(records) == expected_records -@pytest.mark.parametrize("additional_arguments, expected_slices", [ - ({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), - ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}]) -]) +@pytest.mark.parametrize( + "additional_arguments, expected_slices", + [({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}])], +) def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments, expected_slices): if not additional_arguments: requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=read_json("response_survey_ids.json")) @@ -54,11 +58,14 @@ def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments assert list(stream_slices) == expected_slices -@pytest.mark.parametrize("endpoint, records_filename", [ - ("survey_pages", "records_survey_pages.json"), - ("survey_questions", "records_survey_questions.json"), - ("survey_collectors", "records_survey_collectors.json") -]) +@pytest.mark.parametrize( + "endpoint, records_filename", + [ + ("survey_pages", "records_survey_pages.json"), + ("survey_questions", "records_survey_questions.json"), + ("survey_collectors", "records_survey_collectors.json"), + ], +) def test_survey_data(requests_mock, read_records, read_json, endpoint, records_filename): requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/collectors", json=read_json("response_survey_collectors.json")) diff --git a/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/components.py b/airbyte-integrations/connectors/source-the-guardian-api/components.py index 998187314780..8d54333154f2 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/components.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/components.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional import requests + from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement diff --git a/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py index 235d12e640af..1e2ca8425a3f 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py @@ -9,60 +9,57 @@ def create_response(current_page: int, total_pages: int) -> requests.Response: """Helper function to create mock responses""" response = MagicMock(spec=requests.Response) - response.json.return_value = { - "response": { - "currentPage": current_page, - "pages": total_pages - } - } + response.json.return_value = {"response": {"currentPage": current_page, "pages": total_pages}} return response + @pytest.mark.parametrize( "current_page,total_pages,expected_next_page", [ - (1, 5, 2), # First page - (2, 5, 3), # Middle page - (4, 5, 5), # Second to last page - (5, 5, None), # Last page - (1, 1, None), # Single page + (1, 5, 2), # First page + (2, 5, 3), # Middle page + (4, 5, 5), # Second to last page + (5, 5, None), # Last page + (1, 1, None), # Single page ], - ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"] + ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"], ) def test_page_increment(connector_dir, components_module, current_page, total_pages, expected_next_page): """Test the CustomPageIncrement pagination for various page combinations""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Set internal page counter to match current_page paginator._page = current_page - + mock_response = create_response(current_page, total_pages) next_page = paginator.next_page_token(mock_response) assert next_page == expected_next_page, f"Page {current_page} of {total_pages} should get next_page={expected_next_page}" + def test_reset_functionality(components_module): """Test the reset behavior of CustomPageIncrement""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Advance a few pages mock_response = create_response(current_page=1, total_pages=5) paginator.next_page_token(mock_response) paginator.next_page_token(create_response(current_page=2, total_pages=5)) - + # Test reset paginator.reset() assert paginator._page == 1, "Reset should set page back to 1" - + # Verify pagination works after reset next_page = paginator.next_page_token(mock_response) assert next_page == 2, "Should increment to page 2 after reset" diff --git a/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/main.py b/airbyte-integrations/connectors/source-tiktok-marketing/main.py index b523ea1b0fdd..cc7653e249d5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/main.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/main.py @@ -4,5 +4,6 @@ from source_tiktok_marketing.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py index f74710d7d2ff..d7ad3129553a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py @@ -4,6 +4,7 @@ from typing import Any, Iterable, Mapping import dpath.util + from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter from airbyte_cdk.sources.declarative.types import StreamSlice diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py index 628a57a7a89f..21b5ae6eccea 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py @@ -7,7 +7,6 @@ class PerPartitionRecordFilter(RecordFilter): - """ Prepares per partition stream state to be used in the Record Filter condition. Gets current stream state cursor value for stream slice and passes it to condition. diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py index 0b6883e6d13d..ece17f12fc70 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py @@ -18,7 +18,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Mapping[str, Any]: - for metric_key, metric_value in record.get("metrics", {}).items(): if metric_value == self.empty_value: record["metrics"][metric_key] = None diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 8d0b1f1afcc2..6d2655f5ccfb 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream + logger = logging.getLogger("airbyte") DOCUMENTATION_URL = "https://docs.airbyte.com/integrations/sources/tiktok-marketing" SANDBOX_STREAM_NAMES = [ diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py index 9a0abf1bc253..dba9ba324d68 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py @@ -5,6 +5,7 @@ from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template + ADVERTISERS_FILE = "advertisers" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py index 5c32c0b9ea7d..c01481cb119e 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py @@ -7,14 +7,9 @@ class ConfigBuilder: def __init__(self) -> None: self._config: Dict[str, Any] = { - "credentials": { - "auth_type": "oauth2.0", - "access_token": "access token", - "app_id": "11111111111111111111", - "secret": "secret" - }, + "credentials": {"auth_type": "oauth2.0", "access_token": "access token", "app_id": "11111111111111111111", "secret": "secret"}, "start_date": "2024-01-01", - "include_deleted": False + "include_deleted": False, } def with_include_deleted(self) -> "ConfigBuilder": diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py index f3f40d4f3e6e..501f144efcc7 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py @@ -4,13 +4,14 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing class TestCreativeAssetsMusic(TestCase): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py index 19a875a648ef..10dd031fecf4 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py @@ -4,13 +4,14 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing class TestCreativeAssetsPortfolios(TestCase): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py index b0a8f102ce3d..0f72549ef59c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py @@ -4,14 +4,16 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing + EMPTY_LIST_RESPONSE = {"code": 0, "message": "ok", "data": {"list": []}} @@ -88,10 +90,11 @@ class TestAdsReportHourly(TestCase): "real_time_app_install_cost", "app_install", ] + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -113,16 +116,16 @@ def state(self): def mock_response(self, http_mocker: HttpMocker, include_deleted=False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_AD", - "dimensions": '["ad_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_AD", + "dimensions": '["ad_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "ad_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( @@ -132,7 +135,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted=False): ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), ) - query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] + query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( @@ -239,7 +242,7 @@ class TestAdGroupsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -263,16 +266,16 @@ def state(self): def test_basic_read(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -348,17 +351,17 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) filtering = '[{"field_name": "adgroup_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - "filtering": filtering, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + "filtering": filtering, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -380,6 +383,7 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): assert output.records[0].record.data.get("adgroup_id") is not None assert output.records[0].record.data.get("stat_time_hour") is not None + class TestAdvertisersReportsHourly(TestCase): stream_name = "advertisers_reports_hourly" advertiser_id = "872746382648" @@ -536,7 +540,7 @@ class TestCampaignsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -556,18 +560,18 @@ def state(self): .build() ) - def mock_response(self, http_mocker: HttpMocker, include_deleted:bool=False): + def mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_CAMPAIGN", - "dimensions": '["campaign_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_CAMPAIGN", + "dimensions": '["campaign_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "campaign_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py index 1825fa49d955..ce1e6432c576 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py @@ -3,9 +3,6 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.declarative.types import StreamSlice from source_tiktok_marketing import SourceTiktokMarketing from source_tiktok_marketing.components.advertiser_ids_partition_router import ( MultipleAdvertiserIdsPerPartition, @@ -15,13 +12,17 @@ from source_tiktok_marketing.components.semi_incremental_record_filter import PerPartitionRecordFilter from source_tiktok_marketing.components.transformations import TransformEmptyMetrics +from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.declarative.types import StreamSlice + @pytest.mark.parametrize( "config, expected", [ ({"credentials": {"advertiser_id": "11111111111"}}, "11111111111"), ({"environment": {"advertiser_id": "2222222222"}}, "2222222222"), - ({"credentials": {"access_token": "access_token"}}, None) + ({"credentials": {"access_token": "access_token"}}, None), ], ) def test_get_partition_value_from_config(config, expected): @@ -29,8 +30,9 @@ def test_get_partition_value_from_config(config, expected): parent_stream_configs=[MagicMock()], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) actual = router.get_partition_value_from_config() assert actual == expected @@ -39,17 +41,26 @@ def test_get_partition_value_from_config(config, expected): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, - [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_ids": '["2222222222"]', "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], - {"code": 0, "message": "ok", "data": - {"list": [{"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_multiple(config, expected, requests_mock, json_data): @@ -57,23 +68,19 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = MultipleAdvertiserIdsPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_ids", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_ids", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_ids" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_ids", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -81,19 +88,26 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, [{"advertiser_id": "11111111111", "parent_slice": {}}], - None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_id": "11111111111", "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_id": "2222222222", "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_id": "11111111", "parent_slice": {}}, - {"advertiser_id": "22222222", "parent_slice": {}}], - {"code": 0, "message": "ok", - "data": {"list": [ - {"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_id": "11111111", "parent_slice": {}}, {"advertiser_id": "22222222", "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_single(config, expected, requests_mock, json_data): @@ -101,23 +115,19 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = SingleAdvertiserIdPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_id", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_id", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -126,22 +136,22 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): "records, state, slice, expected", [ ( - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], - {}, - {}, - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}] + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], + {}, + {}, + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], ), ( - [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 1}, - [{"advertiser_id": 1, "start_time": "2024-01-02"}] + [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 1}, + [{"advertiser_id": 1, "start_time": "2024-01-02"}], ), ( - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 2}, - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 2}, + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], ), ], ) @@ -150,18 +160,20 @@ def test_record_filter(records, state, slice, expected): record_filter = PerPartitionRecordFilter( config=config, parameters={"partition_field": "advertiser_id"}, - condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}" + condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}", + ) + filtered_records = list( + record_filter.filter_records(records=records, stream_state=state, stream_slice=StreamSlice(partition=slice, cursor_slice={})) ) - filtered_records = list(record_filter.filter_records( - records=records, - stream_state=state, - stream_slice=StreamSlice(partition=slice, cursor_slice={}) - )) assert filtered_records == expected def test_hourly_datetime_based_cursor(): - config = {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, "start_date": "2022-01-01", "end_date": "2022-01-02"} + config = { + "credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, + "start_date": "2022-01-01", + "end_date": "2022-01-02", + } cursor = HourlyDatetimeBasedCursor( start_datetime=MinMaxDatetime(datetime="{{ config.get('start_date', '2016-09-01') }}", datetime_format="%Y-%m-%d", parameters={}), @@ -172,42 +184,33 @@ def test_hourly_datetime_based_cursor(): cursor_field="stat_time_hour", datetime_format="%Y-%m-%d", cursor_datetime_formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"], - parameters={} + parameters={}, ) cursor._cursor = "2022-01-01 00:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] cursor._cursor = "2022-01-01 10:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] @pytest.mark.parametrize( "record, expected", [ + ({"metrics": {"metric_1": "not empty", "metric_2": "-"}}, {"metrics": {"metric_1": "not empty", "metric_2": None}}), + ({"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}), ( - {"metrics": {"metric_1": "not empty", "metric_2": "-"}}, - {"metrics": {"metric_1": "not empty", "metric_2": None}} - ), - ( - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}} - ), - ( - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}} - ), - ( - {}, - {} + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, ), + ({}, {}), ], ) def test_transform_empty_metrics(record, expected): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py index 44db480d1000..f04efc33e7c1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py @@ -6,17 +6,34 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import ConnectorSpecification from source_tiktok_marketing import SourceTiktokMarketing +from airbyte_cdk.models import ConnectorSpecification + @pytest.mark.parametrize( "config, stream_len", [ ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01"}, 36), ({"access_token": "token", "start_date": "2021-01-01", "environment": {"advertiser_id": "1111"}}, 28), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "LIFETIME"}, 15), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "DAY"}, 27), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "LIFETIME", + }, + 15, + ), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "DAY", + }, + 27, + ), ], ) def test_source_streams(config, stream_len): @@ -43,11 +60,27 @@ def config_fixture(): def test_source_check_connection_ok(config, requests_mock): requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/advertiser/info/?page_size=100&advertiser_ids=%5B%22917429327%22%5D", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) logger_mock = MagicMock() assert SourceTiktokMarketing().check_connection(logger_mock, config) == (True, None) @@ -56,20 +89,17 @@ def test_source_check_connection_ok(config, requests_mock): @pytest.mark.parametrize( "json_response, expected_result, expected_message", [ - ({"code": 40105, "message": "Access token is incorrect or has been revoked."}, - (False, "Access token is incorrect or has been revoked."), - None), - ({"code": 40100, "message": "App reaches the QPS limit."}, - None, - 38) - ] + ( + {"code": 40105, "message": "Access token is incorrect or has been revoked."}, + (False, "Access token is incorrect or has been revoked."), + None, + ), + ({"code": 40100, "message": "App reaches the QPS limit."}, None, 38), + ], ) @pytest.mark.usefixtures("mock_sleep") def test_source_check_connection_failed(config, requests_mock, capsys, json_response, expected_result, expected_message): - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_response - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_response) logger_mock = MagicMock() result = SourceTiktokMarketing().check_connection(logger_mock, config) diff --git a/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tplcentral/main.py b/airbyte-integrations/connectors/source-tplcentral/main.py index c5e7b8a95ec4..fb91c7bd8e63 100644 --- a/airbyte-integrations/connectors/source-tplcentral/main.py +++ b/airbyte-integrations/connectors/source-tplcentral/main.py @@ -4,5 +4,6 @@ from source_tplcentral.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py index a20a1c20c81d..40bd46bfe85f 100644 --- a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py @@ -7,10 +7,11 @@ from typing import Any, List, Mapping, MutableMapping, Tuple import requests +from requests.auth import HTTPBasicAuth + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from requests.auth import HTTPBasicAuth from source_tplcentral.streams import Customers, Inventory, Items, Orders, StockDetails, StockSummaries diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py index 97e696a6da7f..15a328c51c51 100644 --- a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py @@ -8,6 +8,7 @@ import arrow import requests + from airbyte_cdk.sources.streams.http import HttpStream from source_tplcentral.util import deep_get, normalize diff --git a/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py index b31950e75404..2a116bd8e350 100644 --- a/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models import SyncMode from source_tplcentral.streams import IncrementalTplcentralStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def config(): diff --git a/airbyte-integrations/connectors/source-trello/components.py b/airbyte-integrations/connectors/source-trello/components.py index dfcb6799f92b..31c67c6ad62e 100644 --- a/airbyte-integrations/connectors/source-trello/components.py +++ b/airbyte-integrations/connectors/source-trello/components.py @@ -14,7 +14,6 @@ @dataclass class OrderIdsPartitionRouter(SubstreamPartitionRouter): def stream_slices(self) -> Iterable[StreamSlice]: - stream_map = {stream_config.stream.name: stream_config.stream for stream_config in self.parent_stream_configs} board_ids = set(self.config.get("board_ids", [])) diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py index 439e5148ef2b..6cc39f099c5d 100644 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py +++ b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py @@ -4,9 +4,10 @@ import pytest -from airbyte_cdk.sources.streams.core import Stream from source_trello.components import OrderIdsPartitionRouter +from airbyte_cdk.sources.streams.core import Stream + class MockStream(Stream): def __init__(self, records): diff --git a/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio/main.py b/airbyte-integrations/connectors/source-twilio/main.py index 0999d1e67f26..972b1826fb0a 100644 --- a/airbyte-integrations/connectors/source-twilio/main.py +++ b/airbyte-integrations/connectors/source-twilio/main.py @@ -4,5 +4,6 @@ from source_twilio.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 26c2269b16af..b2bccf6832de 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Tuple import pendulum + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -48,6 +49,7 @@ VerifyServices, ) + RETENTION_WINDOW_LIMIT = 400 diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 2e82d6639452..415fb3358917 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -10,13 +10,15 @@ import pendulum import requests +from pendulum.datetime import DateTime +from requests.auth import AuthBase + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum.datetime import DateTime -from requests.auth import AuthBase + TWILIO_CHAT_BASE = "https://chat.twilio.com/v2/" TWILIO_CONVERSATION_BASE = "https://conversations.twilio.com/v1/" diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py index 309355222b86..4c365be55842 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py @@ -8,7 +8,6 @@ import pendulum import pytest import requests -from airbyte_cdk.sources.streams.http import HttpStream from freezegun import freeze_time from source_twilio.auth import HttpBasicAuthenticator from source_twilio.source import SourceTwilio @@ -27,6 +26,9 @@ UsageTriggers, ) +from airbyte_cdk.sources.streams.http import HttpStream + + TEST_CONFIG = { "account_sid": "airbyte.io", "auth_token": "secret", @@ -43,7 +45,6 @@ class TestTwilioStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( @@ -60,14 +61,14 @@ def test_data_field(self, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, expected", [ - (Accounts, ['name']), + (Accounts, ["name"]), ], ) def test_changeable_fields(self, stream_cls, expected): - with patch.object(Accounts, "changeable_fields", ['name']): - stream = stream_cls(**self.CONFIG) - result = stream.changeable_fields - assert result == expected + with patch.object(Accounts, "changeable_fields", ["name"]): + stream = stream_cls(**self.CONFIG) + result = stream.changeable_fields + assert result == expected @pytest.mark.parametrize( "stream_cls, expected", @@ -108,12 +109,12 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): with patch.object(TwilioStream, "changeable_fields", ["name"]): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = list(stream.parse_response(response)) - assert result[0]['id'] == expected[0]['id'] + stream = stream_cls(**self.CONFIG) + url = f"{stream.url_base}{stream.path()}" + requests_mock.get(url, json=test_response) + response = requests.get(url) + result = list(stream.parse_response(response)) + assert result[0]["id"] == expected[0]["id"] @pytest.mark.parametrize( "stream_cls, expected", @@ -151,14 +152,13 @@ def test_request_params(self, stream_cls, next_page_token, expected): ("Fri, 11 Dec 2020 04:28:40 +0000", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("2020-12-11T04:28:40Z", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("some_string", {}, "some_string"), - ] + ], ) def test_transform_function(self, original_value, field_schema, expected_value): assert Accounts.custom_transform_function(original_value, field_schema) == expected_value class TestIncrementalTwilioStream: - CONFIG = TEST_CONFIG CONFIG.pop("account_sid") CONFIG.pop("auth_token") @@ -256,7 +256,6 @@ def test_generate_dt_ranges(self, stream_cls, state, expected_dt_ranges): class TestTwilioNestedStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( @@ -299,7 +298,6 @@ def test_stream_slices(self, stream_cls, parent_stream, record, expected): class TestUsageNestedStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-typeform/main.py b/airbyte-integrations/connectors/source-typeform/main.py index 126dc556ff7d..a28ee8adc4c0 100644 --- a/airbyte-integrations/connectors/source-typeform/main.py +++ b/airbyte-integrations/connectors/source-typeform/main.py @@ -4,5 +4,6 @@ from source_typeform.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py index 2ebf804b4940..c5698a824168 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_typeform import SourceTypeform +from airbyte_cdk.entrypoint import launch + def run(): source = SourceTypeform() diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py index 58ec8391d7d7..8bcf7525d561 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py index a841b1d266cf..ee22158b1c58 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py @@ -1,13 +1,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from source_typeform.components import TypeformAuthenticator + from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator -from source_typeform.components import TypeformAuthenticator def test_typeform_authenticator(): config = {"credentials": {"auth_type": "access_token", "access_token": "access_token"}} - oauth_config = {"credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} + oauth_config = { + "credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"} + } class TokenProvider: def get_token(self) -> str: @@ -16,13 +19,13 @@ def get_token(self) -> str: auth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(auth, BearerAuthenticator) oauth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=oauth_config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(oauth, DeclarativeSingleUseRefreshTokenOauth2Authenticator) diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py index c5c3f1508e76..f8f11245c5c3 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py @@ -6,9 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from source_typeform.components import FormIdPartitionRouter, TypeformAuthenticator +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig + + # test cases as a list of tuples (form_ids, parent_stream_configs, expected_slices) test_cases = [ ( diff --git a/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-us-census/components.py b/airbyte-integrations/connectors/source-us-census/components.py index 431d0794f7ad..73087fa6c63a 100644 --- a/airbyte-integrations/connectors/source-us-census/components.py +++ b/airbyte-integrations/connectors/source-us-census/components.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler @@ -116,7 +117,6 @@ class USCensusErrorHandler(DefaultErrorHandler): """ def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]]) -> ErrorResolution: - if self.response_filters: for response_filter in self.response_filters: matched_error_resolution = response_filter.matches(response_or_exception=response_or_exception) diff --git a/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-webflow/main.py b/airbyte-integrations/connectors/source-webflow/main.py index 4d481e07151b..5f67a19aa9a7 100644 --- a/airbyte-integrations/connectors/source-webflow/main.py +++ b/airbyte-integrations/connectors/source-webflow/main.py @@ -4,5 +4,6 @@ from source_webflow.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py index f7e3daa64937..4eacbe1bfcd8 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -15,6 +16,7 @@ from .auth import WebflowTokenAuthenticator from .webflow_to_airbyte_mapping import WebflowToAirbyteMapping + """ This module is used for pulling the contents of "collections" out of Webflow, which is a CMS for hosting websites. A Webflow collection may be a group of items such as "Blog Posts", "Blog Authors", etc. @@ -201,7 +203,6 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - # Webflow default pagination is 100, for debugging pagination we set this to a low value. # This should be set back to 100 for production params = {"limit": 100} @@ -256,7 +257,6 @@ def get_json_schema(self) -> Mapping[str, Any]: class SourceWebflow(AbstractSource): - """This is the main class that defines the methods that will be called by Airbyte infrastructure""" @staticmethod diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py index ea40dc0ab320..4b6061984649 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py @@ -4,7 +4,6 @@ class WebflowToAirbyteMapping: - """ The following disctionary is used for dynamically pulling the schema from Webflow, and mapping it to an Airbyte-compatible json-schema Webflow: https://developers.webflow.com/#get-collection-with-full-schema diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py index aaeb7f6c2529..a56a495fcd92 100644 --- a/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-xero/main.py b/airbyte-integrations/connectors/source-xero/main.py index d765f10d2093..40c719373836 100644 --- a/airbyte-integrations/connectors/source-xero/main.py +++ b/airbyte-integrations/connectors/source-xero/main.py @@ -4,5 +4,6 @@ from source_xero.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-xero/source_xero/components.py b/airbyte-integrations/connectors/source-xero/source_xero/components.py index 2294372e6ab5..a9e4a6655cd3 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/components.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/components.py @@ -9,6 +9,7 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-xero/source_xero/run.py b/airbyte-integrations/connectors/source-xero/source_xero/run.py index fb8d5955af03..99bb37a63595 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/run.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_xero import SourceXero +from airbyte_cdk.entrypoint import launch + def run(): source = SourceXero() diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index 3209646cb044..7da885250cc5 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py index a2dcb6a4541a..1f8b9a330d54 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py @@ -4,10 +4,11 @@ from typing import Any, Mapping -from airbyte_cdk.sources.streams import Stream from pytest import fixture from source_xero.source import SourceXero +from airbyte_cdk.sources.streams import Stream + @fixture(name="config_pass") def config_fixture(): diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py index 13e3b37c9f12..17795ec5716f 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py @@ -4,13 +4,16 @@ import datetime -from airbyte_cdk.models import SyncMode from conftest import get_stream_by_name from source_xero.components import ParseDates +from airbyte_cdk.models import SyncMode + def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -22,7 +25,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py index ee4687b936ef..3765b99482f8 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py @@ -22,6 +22,7 @@ def test_check_connection_failed(bad_config, requests_mock): assert check_succeeded is False assert error == "" or "none" in error.lower() + def test_streams_count(config_pass): source = SourceXero() streams = source.streams(config_pass) diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py index a0bdf0e11610..6dffe0aeb3d2 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py @@ -4,13 +4,16 @@ import datetime -from airbyte_cdk.models import SyncMode from conftest import get_stream_by_name from source_xero.components import ParseDates +from airbyte_cdk.models import SyncMode + def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -26,8 +29,8 @@ def test_request_params(config_pass): def test_request_headers(config_pass): bank_transactions = get_stream_by_name("bank_transactions", config_pass) - expected_headers = {'Xero-Tenant-Id': 'goodone', 'Accept': 'application/json'} - assert bank_transactions.retriever.requester.get_request_headers() == expected_headers + expected_headers = {"Xero-Tenant-Id": "goodone", "Accept": "application/json"} + assert bank_transactions.retriever.requester.get_request_headers() == expected_headers def test_http_method(config_pass): @@ -38,7 +41,7 @@ def test_http_method(config_pass): def test_ignore_forbidden(requests_mock, config_pass): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{ "message": "Forbidden resource"}]) + requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{"message": "Forbidden resource"}]) stream = get_stream_by_name("bank_transactions", config_pass) records = [] @@ -52,7 +55,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/main.py b/airbyte-integrations/connectors/source-yandex-metrica/main.py index a84b23e0a261..ad216843240a 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/main.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/main.py @@ -4,5 +4,6 @@ from source_yandex_metrica.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py index 874910073f0e..e917e2fa28ed 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py @@ -6,11 +6,13 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from .streams import Sessions, Views, YandexMetricaStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py index 957030f23f9d..65592bdb7154 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py @@ -12,11 +12,13 @@ import pendulum import requests +from pendulum import DateTime + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams.core import IncrementalMixin, StreamData from airbyte_cdk.sources.streams.http import HttpStream -from pendulum import DateTime + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py index 3993932b1c69..bf5adface5a2 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py @@ -8,6 +8,7 @@ import pytest from source_yandex_metrica.source import SourceYandexMetrica + logger = logging.getLogger("test_source") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py index afd233b11790..cec63d87d2d2 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py @@ -3,9 +3,11 @@ # -from airbyte_cdk.models import SyncMode from source_yandex_metrica.streams import Sessions +from airbyte_cdk.models import SyncMode + + EXPECTED_RECORDS = [ {"watchID": "00000000", "dateTime": "2022-09-01T12:00:00+00:00"}, {"watchID": "00000001", "dateTime": "2022-08-01T12:00:10+00:00"}, diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-younium/components.py b/airbyte-integrations/connectors/source-younium/components.py index 7b9bcaa42a5f..f202ba2a73ec 100644 --- a/airbyte-integrations/connectors/source-younium/components.py +++ b/airbyte-integrations/connectors/source-younium/components.py @@ -7,10 +7,12 @@ from typing import Any, Mapping, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response # The Bearer token generated by server-to-server token will expire in one hour diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/main.py b/airbyte-integrations/connectors/source-youtube-analytics/main.py index f2542cccc965..959f6b271454 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/main.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/main.py @@ -4,5 +4,6 @@ from source_youtube_analytics.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py index 4a9041f1f078..cd6af9e03d9b 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py @@ -12,6 +12,7 @@ import pendulum import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream diff --git a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py index dcac2eb7e98a..9744592d3ba5 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py @@ -6,9 +6,10 @@ import os from unittest.mock import MagicMock -from airbyte_cdk.sources.streams.http.auth.core import NoAuth from source_youtube_analytics.source import SourceYoutubeAnalytics +from airbyte_cdk.sources.streams.http.auth.core import NoAuth + def test_check_connection(requests_mock): access_token = "token" diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py index 13626e17bbbc..bbcb318d24a8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py index 43ce950d77ca..72132012aaed 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main.py b/airbyte-integrations/connectors/source-zendesk-chat/main.py index c2c8d74d092a..4548150f4115 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/main.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/main.py @@ -4,5 +4,6 @@ from source_zendesk_chat.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py index 2dffe978edfb..28e6de3dd4fc 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index 9c3eb3109f52..cce977551ac7 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py index 284325c12e3b..6cbdc2694790 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 2b0540f7cd8f..bbaafa375146 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py index c48196cfa1ed..8083a1cc5e0d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py @@ -12,25 +12,14 @@ def config() -> Mapping[str, Any]: return { "start_date": "2020-10-01T00:00:00Z", "subdomain": "", - "credentials": { - "credentials": "access_token", - "access_token": "__access_token__" - } + "credentials": {"credentials": "access_token", "access_token": "__access_token__"}, } @pytest.fixture def bans_stream_record() -> Mapping[str, Any]: return { - "ip_address": [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - } - ], + "ip_address": [{"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}], "visitor": [ { "type": "visitor", @@ -38,28 +27,22 @@ def bans_stream_record() -> Mapping[str, Any]: "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", } - ] + ], } @pytest.fixture def bans_stream_record_extractor_expected_output() -> List[Mapping[str, Any]]: return [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - }, + {"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}, { "type": "visitor", "id": 4444, "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", }, ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py index 60ae75a17da1..d33b2302e161 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py @@ -8,8 +8,8 @@ def test_bans_stream_record_extractor( config, - requests_mock, - bans_stream_record, + requests_mock, + bans_stream_record, bans_stream_record_extractor_expected_output, ) -> None: test_url = f"https://{config['subdomain']}.zendesk.com/api/v2/chat/bans" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py index 9557a312b635..75dad4a8f0d5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -9,26 +9,24 @@ def _get_cursor(config) -> ZendeskChatIdIncrementalCursor: return ZendeskChatIdIncrementalCursor( - config = config, - cursor_field = "id", - field_name = "since_id", - parameters = {}, + config=config, + cursor_field="id", + field_name="since_id", + parameters={}, ) @pytest.mark.parametrize( "stream_state, expected_cursor_value, expected_state_value", [ - ({"id": 10}, 10, {'id': 10}), + ({"id": 10}, 10, {"id": 10}), ], - ids=[ - "SET Initial State and GET State" - ] + ids=["SET Initial State and GET State"], ) def test_id_incremental_cursor_set_initial_state_and_get_stream_state( - config, + config, stream_state, - expected_cursor_value, + expected_cursor_value, expected_state_value, ) -> None: cursor = _get_cursor(config) @@ -44,17 +42,14 @@ def test_id_incremental_cursor_set_initial_state_and_get_stream_state( ({"id": 123}, 123), ({"id": 456}, 456), ], - ids=[ - "first", - "second" - ] + ids=["first", "second"], ) def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: cursor = _get_cursor(config) cursor.observe(stream_slice={}, record=test_record) cursor.close_slice(stream_slice={}) assert cursor._cursor == expected - + @pytest.mark.parametrize( "stream_state, input_slice, expected", @@ -62,17 +57,14 @@ def test_id_incremental_cursor_close_slice(config, test_record, expected) -> Non ({}, {"id": 1}, {}), ({"id": 2}, {"id": 1}, {"since_id": 2}), ], - ids=[ - "No State", - "With State" - ] + ids=["No State", "With State"], ) def test_id_incremental_cursor_get_request_params(config, stream_state, input_slice, expected) -> None: cursor = _get_cursor(config) if stream_state: cursor.set_initial_state(stream_state) assert cursor.get_request_params(stream_slice=input_slice) == expected - + @pytest.mark.parametrize( "stream_state, record, expected", @@ -85,7 +77,7 @@ def test_id_incremental_cursor_get_request_params(config, stream_state, input_sl "No State", "With State > Record value", "With State < Record value", - ] + ], ) def test_id_incremental_cursor_should_be_synced(config, stream_state, record, expected) -> None: cursor = _get_cursor(config) @@ -107,7 +99,7 @@ def test_id_incremental_cursor_should_be_synced(config, stream_state, record, ex "First < Second - should not be synced", "Has First but no Second - should be synced", "Has no First and has no Second - should not be synced", - ] + ], ) def test_id_incremental_cursor_is_greater_than_or_equal(config, first_record, second_record, expected) -> None: cursor = _get_cursor(config) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py index ddf84932adc0..48f06329adbd 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py @@ -10,19 +10,16 @@ def _get_paginator(config, id_field) -> ZendeskChatIdOffsetIncrementPaginationStrategy: return ZendeskChatIdOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - id_field = id_field, - parameters = {}, + config=config, + page_size=1, + id_field=id_field, + parameters={}, ) @pytest.mark.parametrize( "id_field, last_records, expected", - [ - ("id", [{"id": 1}], 2), - ("id", [], None) - ], + [("id", [{"id": 1}], 2), ("id", [], None)], ) def test_id_offset_increment_pagination_next_page_token(requests_mock, config, id_field, last_records, expected) -> None: paginator = _get_paginator(config, id_field) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py index 61d793cbc9f9..10838351b5c5 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py @@ -10,18 +10,18 @@ def _get_paginator(config, time_field_name) -> ZendeskChatTimeOffsetIncrementPaginationStrategy: return ZendeskChatTimeOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - time_field_name = time_field_name, - parameters = {}, + config=config, + page_size=1, + time_field_name=time_field_name, + parameters={}, ) @pytest.mark.parametrize( "time_field_name, response, last_records, expected", [ - ("end_time", {"chats":[{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), - ("end_time", {"chats":[], "end_time": 3}, [], None), + ("end_time", {"chats": [{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), + ("end_time", {"chats": [], "end_time": 3}, [], None), ], ) def test_time_offset_increment_pagination_next_page_token(requests_mock, config, time_field_name, response, last_records, expected) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py index a98cc8283e93..2528f7a5db7e 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -4,23 +4,24 @@ import pytest -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from source_zendesk_chat.components.timestamp_based_cursor import ZendeskChatTimestampCursor +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType + def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampCursor: cursor = ZendeskChatTimestampCursor( - start_datetime = "2020-10-01T00:00:00Z", - cursor_field = cursor_field, - datetime_format = "%s", - config = config, - parameters = {}, - use_microseconds = f"{{{ {use_microseconds} }}}", + start_datetime="2020-10-01T00:00:00Z", + cursor_field=cursor_field, + datetime_format="%s", + config=config, + parameters={}, + use_microseconds=f"{{{ {use_microseconds} }}}", ) # patching missing parts cursor.start_time_option = RequestOption( - field_name = cursor_field, - inject_into = RequestOptionType.request_parameter, + field_name=cursor_field, + inject_into=RequestOptionType.request_parameter, parameters={}, ) return cursor @@ -29,25 +30,25 @@ def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampC @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), + (True, {"start_time": 1}, {"start_time": 1000000}), ], ) def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) test_result = cursor.add_microseconds({}, input_slice) assert test_result == expected - + @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), - (False, {"start_time": 1}, {'start_time': 1}), + (True, {"start_time": 1}, {"start_time": 1000000}), + (False, {"start_time": 1}, {"start_time": 1}), ], ids=[ "WITH `use_microseconds`", "WITHOUT `use_microseconds`", - ] + ], ) def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py index 9e6409236281..a612c74fc689 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-support/main.py b/airbyte-integrations/connectors/source-zendesk-support/main.py index 88eed5ec56af..4577f470a2dc 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/main.py +++ b/airbyte-integrations/connectors/source-zendesk-support/main.py @@ -4,5 +4,6 @@ from source_zendesk_support.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py index 1f631c62a130..4d205ed5e83b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType @@ -33,7 +34,9 @@ def get_request_params( start_time = stream_slice.get(self._partition_field_start.eval(self.config)) options[self.start_time_option.field_name.eval(config=self.config)] = [start_time] # type: ignore # field_name is always casted to an interpolated string if self.end_time_option and self.end_time_option.inject_into == option_type: - options[self.end_time_option.field_name.eval(config=self.config)].append(stream_slice.get(self._partition_field_end.eval(self.config))) # type: ignore # field_name is always casted to an interpolated string + options[self.end_time_option.field_name.eval(config=self.config)].append( + stream_slice.get(self._partition_field_end.eval(self.config)) + ) # type: ignore # field_name is always casted to an interpolated string return options diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py index 0df09fb2f375..d040b0f77975 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py @@ -7,10 +7,11 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_zendesk_support import SourceZendeskSupport diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py index 15d04a0d21dc..f7d4746facf6 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState @@ -29,6 +30,7 @@ UserSettingsStream, ) + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 911c090f5545..8cc0026bd99f 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -13,6 +13,7 @@ import pendulum import pytz import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration @@ -24,6 +25,7 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException + DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY: str = "_last_end_time" END_OF_STREAM_KEY: str = "end_of_stream" @@ -518,7 +520,6 @@ def migrate(self, stream_state: Optional[Mapping[str, Any]]) -> Mapping[str, Any class TicketMetrics(SourceZendeskSupportStream): - name = "ticket_metrics" cursor_field = "_ab_updated_at" should_checkpoint = False @@ -583,7 +584,6 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, class StatelessTicketMetrics(FullRefreshZendeskSupportStream): - response_list_name: str = "ticket_metrics" cursor_field: str = "updated_at" should_checkpoint = False @@ -631,7 +631,6 @@ def _get_updated_state(self, current_stream_state: Mapping[str, Any], latest_rec class StatefulTicketMetrics(HttpSubStream, IncrementalZendeskSupportStream): - response_list_name: str = "ticket_metric" _state_cursor_field: str = "_ab_updated_at" _legacy_cursor_field: str = "generated_timestamp" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index c3d9c1c98188..27703dc2ddbb 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -2,4 +2,5 @@ import os + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py index 5f02c2b74fbb..a2f658a70079 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py @@ -1,9 +1,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum +from pendulum.datetime import DateTime + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath -from pendulum.datetime import DateTime from .utils import datetime_to_string from .zs_requests import ( @@ -81,30 +82,28 @@ def given_post_comments( ) return post_comments_record_builder + def given_tickets(http_mocker: HttpMocker, start_date: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: """ Tickets requests setup """ - tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field( - FieldPath("generated_timestamp"), start_date.int_timestamp - ) + tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field(FieldPath("generated_timestamp"), start_date.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder -def given_tickets_with_state(http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: + +def given_tickets_with_state( + http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator +) -> TicketsRecordBuilder: """ Tickets requests setup """ tickets_record_builder = TicketsRecordBuilder.tickets_record().with_cursor(cursor_value.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder @@ -114,24 +113,20 @@ def given_groups_with_later_records( http_mocker: HttpMocker, updated_at_value: DateTime, later_record_time_delta: pendulum.duration, - api_token_authenticator: ApiTokenAuthenticator + api_token_authenticator: ApiTokenAuthenticator, ) -> GroupsRecordBuilder: """ Creates two group records one with a specific cursor value and one that has a later cursor value based on the provided timedelta. This is intended to create multiple records with different times which can be used to test functionality like semi-incremental record filtering """ - groups_record_builder = GroupsRecordBuilder.groups_record().with_field( - FieldPath("updated_at"), datetime_to_string(updated_at_value) - ) + groups_record_builder = GroupsRecordBuilder.groups_record().with_field(FieldPath("updated_at"), datetime_to_string(updated_at_value)) later_groups_record_builder = GroupsRecordBuilder.groups_record().with_field( FieldPath("updated_at"), datetime_to_string(updated_at_value + later_record_time_delta) ) http_mocker.get( - GroupsRequestBuilder.groups_endpoint(api_token_authenticator) - .with_page_size(100) - .build(), + GroupsRequestBuilder.groups_endpoint(api_token_authenticator).with_page_size(100).build(), GroupsResponseBuilder.groups_response().with_record(groups_record_builder).with_record(later_groups_record_builder).build(), ) return groups_record_builder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py index 032b9ed1d353..e672521e1501 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py @@ -4,6 +4,7 @@ from unittest import TestCase import pendulum + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.state_builder import StateBuilder @@ -13,6 +14,7 @@ from .utils import datetime_to_string, read_stream, string_to_datetime from .zs_requests.request_authenticators import ApiTokenAuthenticator + _NOW = datetime.now(timezone.utc) @@ -62,9 +64,7 @@ def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_reco api_token_authenticator, ) - state_value = { - "updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50)) - } + state_value = {"updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50))} state = StateBuilder().with_stream_state("groups", state_value).build() diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py index ffe141cdfcb4..1755a17c7f6e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostCommentVotesResponseBuilder from .zs_responses.records import PostCommentVotesRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -96,7 +97,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -126,7 +133,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py index a9d2d825c5bb..a2d140bcf2c3 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostsCommentsResponseBuilder from .zs_responses.records import PostsCommentsRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -84,7 +85,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +116,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py index a2617b5a07df..8ca10b18980d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostsVotesResponseBuilder from .zs_responses.records import PostsVotesRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -84,7 +85,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +116,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py index 699d09742f5d..c5e2dda402cd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py @@ -4,6 +4,7 @@ import freezegun import pendulum + from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath @@ -17,20 +18,21 @@ from .zs_responses import TicketMetricsResponseBuilder from .zs_responses.records import TicketMetricsRecordBuilder + _NOW = pendulum.now(tz="UTC") _TWO_YEARS_AGO_DATETIME = _NOW.subtract(years=2) + @freezegun.freeze_time(_NOW.isoformat()) class TestTicketMetricsIncremental(TestCase): - @property def _config(self): return ( ConfigBuilder() - .with_basic_auth_credentials("user@example.com", "password") - .with_subdomain("d3v-airbyte") - .with_start_date(_TWO_YEARS_AGO_DATETIME) - .build() + .with_basic_auth_credentials("user@example.com", "password") + .with_subdomain("d3v-airbyte") + .with_start_date(_TWO_YEARS_AGO_DATETIME) + .build() ) def _get_authenticator(self, config): @@ -46,7 +48,7 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec http_mocker.get( TicketMetricsRequestBuilder.stateless_ticket_metrics_endpoint(api_token_authenticator).with_page_size(100).build(), - TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build() + TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) @@ -55,7 +57,6 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec assert output.most_recent_state.stream_descriptor.name == "ticket_metrics" assert output.most_recent_state.stream_state.__dict__ == {"_ab_updated_at": pendulum.parse(record_updated_at).int_timestamp} - @HttpMocker() def test_given_state_and_successful_sync_when_read_then_return_record(self, http_mocker): api_token_authenticator = self._get_authenticator(self._config) @@ -63,16 +64,20 @@ def test_given_state_and_successful_sync_when_read_then_return_record(self, http state_cursor_value = pendulum.now(tz="UTC").subtract(days=2).int_timestamp state = StateBuilder().with_stream_state("ticket_metrics", state={"_ab_updated_at": state_cursor_value}).build() record_cursor_value = pendulum.now(tz="UTC").subtract(days=1) - tickets_records_builder = given_tickets_with_state(http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value,api_token_authenticator) + tickets_records_builder = given_tickets_with_state( + http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value, api_token_authenticator + ) ticket = tickets_records_builder.build() - ticket_metrics_first_record_builder = TicketMetricsRecordBuilder.stateful_ticket_metrics_record().with_field( - FieldPath("ticket_id"), ticket["id"] - ).with_cursor(ticket["generated_timestamp"]) + ticket_metrics_first_record_builder = ( + TicketMetricsRecordBuilder.stateful_ticket_metrics_record() + .with_field(FieldPath("ticket_id"), ticket["id"]) + .with_cursor(ticket["generated_timestamp"]) + ) http_mocker.get( TicketMetricsRequestBuilder.stateful_ticket_metrics_endpoint(api_token_authenticator, ticket["id"]).build(), - TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build() + TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py index d3185b94d95e..aa19dc6ef9ca 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py @@ -4,13 +4,13 @@ from typing import Any, Dict, List, Optional import pendulum -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage +from pendulum.datetime import DateTime +from source_zendesk_support import SourceZendeskSupport + +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from pendulum.datetime import DateTime -from source_zendesk_support import SourceZendeskSupport def read_stream( diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py index a185dd225a2f..8d72e988f203 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class ZendeskSupportBaseRequestBuilder(ZendeskSuppportRequestBuilder): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py index 3d224cd79456..94629bb15526 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py @@ -10,7 +10,9 @@ class PostCommentVotesRequestBuilder(ZendeskSupportBaseRequestBuilder): @classmethod - def post_comment_votes_endpoint(cls, authenticator: Authenticator, post_id: int, post_comment_id: int) -> "PostCommentVotesRequestBuilder": + def post_comment_votes_endpoint( + cls, authenticator: Authenticator, post_id: int, post_comment_id: int + ) -> "PostCommentVotesRequestBuilder": return cls("d3v-airbyte", f"community/posts/{post_id}/comments/{post_comment_id}/votes").with_authenticator(authenticator) def __init__(self, subdomain: str, resource: str) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py index 08d634cea66c..adef1134aadb 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py @@ -10,6 +10,7 @@ from source_zendesk_support.source import SourceZendeskSupport from source_zendesk_support.streams import Users + _ANY_ATTEMPT_COUNT = 10 diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py index 6398c165cc46..c0b61a2d29f4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py @@ -4,13 +4,14 @@ import pytest import requests -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from source_zendesk_support.components import ( ZendeskSupportAttributeDefinitionsExtractor, ZendeskSupportAuditLogsIncrementalSync, ZendeskSupportExtractorEvents, ) +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType + @pytest.mark.parametrize( "stream_state, stream_slice, next_page_token, expected_params", diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index ae922ca7ffc8..7c84300b4d87 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -16,8 +16,6 @@ import pytest import pytz import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from source_zendesk_support.source import BasicApiTokenAuthenticator, SourceZendeskSupport from source_zendesk_support.streams import ( DATETIME_FORMAT, @@ -65,6 +63,10 @@ from test_data.data import TICKET_EVENTS_STREAM_RESPONSE from utils import read_full_refresh +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + + TICKET_SUBSTREAMS = [StatefulTicketMetrics] # prepared config @@ -143,7 +145,9 @@ def test_convert_config2stream_args(config): @freezegun.freeze_time("2022-01-01") def test_default_start_date(): - result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args(TEST_CONFIG_WITHOUT_START_DATE) + result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args( + TEST_CONFIG_WITHOUT_START_DATE + ) assert result["start_date"] == "2020-01-01T00:00:00Z" @@ -1047,14 +1051,14 @@ def test_read_non_json_error(requests_mock, caplog): read_full_refresh(stream) assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") -class TestTicketMetrics: +class TestTicketMetrics: @pytest.mark.parametrize( - "state, expected_implemented_stream", - [ - ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - ({}, StatelessTicketMetrics), - ] + "state, expected_implemented_stream", + [ + ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + ({}, StatelessTicketMetrics), + ], ) def test_get_implemented_stream(self, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1062,12 +1066,12 @@ def test_get_implemented_stream(self, state, expected_implemented_stream): assert isinstance(implemented_stream, expected_implemented_stream) @pytest.mark.parametrize( - "sync_mode, state, expected_implemented_stream", - [ - (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - (SyncMode.full_refresh, {}, StatelessTicketMetrics), - (SyncMode.incremental, {}, StatelessTicketMetrics), - ] + "sync_mode, state, expected_implemented_stream", + [ + (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + (SyncMode.full_refresh, {}, StatelessTicketMetrics), + (SyncMode.incremental, {}, StatelessTicketMetrics), + ], ) def test_stream_slices(self, sync_mode, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1081,7 +1085,12 @@ class TestStatefulTicketMetrics: [ ( {}, - {"tickets": [{"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}]}, + { + "tickets": [ + {"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + ] + }, [ {"ticket_id": "13", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"ticket_id": "80", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, @@ -1108,8 +1117,7 @@ def test_stream_slices(self, requests_mock, stream_state, response, expected_sli def test_read_with_error(self, requests_mock): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) requests_mock.get( - f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", - json={"error": "RecordNotFound", "description": "Not found"} + f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", json={"error": "RecordNotFound", "description": "Not found"} ) records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"ticket_id": "13"})) @@ -1119,12 +1127,12 @@ def test_read_with_error(self, requests_mock): @pytest.mark.parametrize( "status_code, response_action", ( - (200, ResponseAction.SUCCESS), - (404, ResponseAction.IGNORE), - (403, ResponseAction.IGNORE), - (500, ResponseAction.RETRY), - (429, ResponseAction.RATE_LIMITED), - ) + (200, ResponseAction.SUCCESS), + (404, ResponseAction.IGNORE), + (403, ResponseAction.IGNORE), + (500, ResponseAction.RETRY), + (429, ResponseAction.RATE_LIMITED), + ), ) def test_should_retry(self, status_code: int, response_action: bool): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) @@ -1135,37 +1143,62 @@ def test_should_retry(self, status_code: int, response_action: bool): @pytest.mark.parametrize( "current_stream_state, record_cursor_value, expected", [ - ({ "_ab_updated_at": 1727334000}, 1727420400, { "_ab_updated_at": 1727420400}), - ({ "_ab_updated_at": 1727334000}, 1700000000, { "_ab_updated_at": 1727334000}), - ] + ({"_ab_updated_at": 1727334000}, 1727420400, {"_ab_updated_at": 1727420400}), + ({"_ab_updated_at": 1727334000}, 1700000000, {"_ab_updated_at": 1727334000}), + ], ) def test_get_updated_state(self, current_stream_state, record_cursor_value, expected): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) - latest_record = { "id": 1, "_ab_updated_at": record_cursor_value} + latest_record = {"id": 1, "_ab_updated_at": record_cursor_value} output_state = stream._get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) assert output_state == expected class TestStatelessTicketMetrics: @pytest.mark.parametrize( - "start_date, response, expected", - [ - ( - "2023-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp}, - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + "start_date, response, expected", + [ + ( + "2023-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ), - ( - "2024-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + }, + [ + { + "id": 1, + "ticket_id": 999, + "updated_at": "2023-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp, + }, + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + }, + ], + ), + ( + "2024-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ) - ] + }, + [ + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } + ], + ), + ], ) def test_parse_response(self, requests_mock, start_date, response, expected): stream_args = copy.deepcopy(STREAM_ARGS) @@ -1176,25 +1209,24 @@ def test_parse_response(self, requests_mock, start_date, response, expected): output = list(stream.parse_response(test_response, {})) assert expected == output - @pytest.mark.parametrize( - "has_more, expected", - [ - (True, {"page[after]": "nextpagecursor"}), - (False, None) - ] - ) + @pytest.mark.parametrize("has_more, expected", [(True, {"page[after]": "nextpagecursor"}), (False, None)]) def test_next_page_token(self, mocker, has_more, expected): stream = StatelessTicketMetrics(**STREAM_ARGS) ticket_metrics_response = mocker.Mock() - ticket_metrics_response.json.return_value = {"meta": { "after_cursor": "nextpagecursor", "has_more": has_more}} + ticket_metrics_response.json.return_value = {"meta": {"after_cursor": "nextpagecursor", "has_more": has_more}} result = stream.next_page_token(response=ticket_metrics_response) assert expected == result def test_get_updated_state(self): stream = StatelessTicketMetrics(**STREAM_ARGS) - stream._most_recently_updated_record = {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + stream._most_recently_updated_record = { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } output_state = stream._get_updated_state(current_stream_state={}, latest_record={}) - expected_state = { "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + expected_state = {"_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} assert output_state == expected_state @@ -1236,13 +1268,7 @@ def test_validate_response_ticket_audits_handle_empty_response(audits_response, assert stream._validate_response(response_mock, {}) == expected -@pytest.mark.parametrize( - "initial_state_cursor_field", - [ - "generated_timestamp", - "_ab_updated_at" - ] -) +@pytest.mark.parametrize("initial_state_cursor_field", ["generated_timestamp", "_ab_updated_at"]) def test_ticket_metrics_state_migrataion(initial_state_cursor_field): state_migrator = TicketMetricsStateMigration() initial_state = {initial_state_cursor_field: 1672531200} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/components.py b/airbyte-integrations/connectors/source-zendesk-talk/components.py index d8030841afe2..f9f3b12732a0 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/components.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenloop/main.py b/airbyte-integrations/connectors/source-zenloop/main.py index dd3a6687740e..502dfc3af95f 100644 --- a/airbyte-integrations/connectors/source-zenloop/main.py +++ b/airbyte-integrations/connectors/source-zenloop/main.py @@ -4,5 +4,6 @@ from source_zenloop.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py index 2f85e4b1b8e1..a2bec3781798 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py @@ -12,7 +12,6 @@ @dataclass class ZenloopPartitionRouter(SubstreamPartitionRouter): - config: Config def stream_slices(self) -> Iterable[StreamSlice]: diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py index 15a603417b4b..70f571dc0543 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py index 00f466edd7e3..53ece70f20ff 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py @@ -9,11 +9,11 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpStream class ZenloopStream(HttpStream, ABC): - url_base = "https://api.zenloop.com/v1/" extra_params = None has_date_param = False @@ -58,7 +58,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class ChildStreamMixin: - parent_stream_class: Optional[ZenloopStream] = None def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: diff --git a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py index 472f2799e63d..76d7794113f8 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py +++ b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py @@ -11,6 +11,7 @@ import requests from source_zoho_crm.streams import IncrementalZohoCrmStream, ZohoStreamFactory + HERE = Path(__file__).parent diff --git a/airbyte-integrations/connectors/source-zoho-crm/main.py b/airbyte-integrations/connectors/source-zoho-crm/main.py index 2cf82bb23bf9..d04f943d86bf 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/main.py +++ b/airbyte-integrations/connectors/source-zoho-crm/main.py @@ -4,5 +4,6 @@ from source_zoho_crm.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index 15425f380be4..b9ebc28e9f6a 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", ] diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py index edb92e2c8e5d..abaf9d55c761 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py @@ -11,6 +11,7 @@ from .auth import ZohoOauth2Authenticator + logger = logging.getLogger(__name__) diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py index f6cde3b11210..6f47fddbee11 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Mapping, Tuple import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py index d915dc05f2b9..05ca883b8f7e 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_zoho_crm import SourceZohoCrm +from airbyte_cdk.entrypoint import launch + def run(): source = SourceZohoCrm() diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py index d6ead32026f8..f5f4ee55e470 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py @@ -10,6 +10,7 @@ from .api import ZohoAPI from .streams import ZohoStreamFactory + if TYPE_CHECKING: # This is a workaround to avoid circular import in the future. # TYPE_CHECKING is False at runtime, but True when system performs type checking diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py index ad6b26868396..53df5ab1e428 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py @@ -11,12 +11,14 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpStream from .api import ZohoAPI from .exceptions import IncompleteMetaDataException, UnknownDataTypeException from .types import FieldMeta, ModuleMeta, ZohoPickListItem + # 204 and 304 status codes are valid successful responses, # but `.json()` will fail because the response body is empty EMPTY_BODY_STATUSES = (HTTPStatus.NO_CONTENT, HTTPStatus.NOT_MODIFIED) diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py index 31ad48a1a41c..ca796e341b23 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py @@ -6,6 +6,7 @@ import pytest + TestCase = namedtuple( "TestCase", ("json_type", "data_type", "length", "decimal_place", "api_name", "pick_list_values", "autonumber", "expected_values") ) diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py index 0fea743d100d..a32f0adf2140 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py @@ -6,6 +6,7 @@ from source_zoho_crm.auth import ZohoOauth2Authenticator + authenticator = ZohoOauth2Authenticator("http://dummy.url/oauth/v2/token", "client_id", "client_secret", "refresh_token") diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py index 3774064c60a2..e761736c840b 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py @@ -5,9 +5,10 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import SyncMode from source_zoho_crm.streams import IncrementalZohoCrmStream as BaseIncrementalZohoCrmStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def stream_factory(mocker): diff --git a/airbyte-integrations/connectors/source-zoom/components.py b/airbyte-integrations/connectors/source-zoom/components.py index 00214c737833..ef268f774550 100644 --- a/airbyte-integrations/connectors/source-zoom/components.py +++ b/airbyte-integrations/connectors/source-zoom/components.py @@ -9,10 +9,12 @@ from typing import Any, Mapping, Optional, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response # The Bearer token generated by server-to-server token will expire in one hour diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py index 82823254d266..fbed37992cb4 100644 --- a/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/tools/bin/cleanup-workflow-runs.py b/tools/bin/cleanup-workflow-runs.py index d2ef9a38cbbc..f022934571d1 100644 --- a/tools/bin/cleanup-workflow-runs.py +++ b/tools/bin/cleanup-workflow-runs.py @@ -10,6 +10,7 @@ from github import Github + DAYS_TO_KEEP_ORPHANED_JOBS = 90 diff --git a/tools/bin/identify-dormant-workflows.py b/tools/bin/identify-dormant-workflows.py index 5dfa954c6056..40a62d426af3 100644 --- a/tools/bin/identify-dormant-workflows.py +++ b/tools/bin/identify-dormant-workflows.py @@ -13,6 +13,7 @@ from slack_sdk import WebClient from slack_sdk.errors import SlackApiError + DAYS_TO_KEEP_ORPHANED_JOBS = 90 SLACK_CHANNEL_FOR_NOTIFICATIONS = "infra-alerts" @@ -97,7 +98,6 @@ def main(): print(message) if slack_token: - print("Sending Slack notification...") client = WebClient(slack_token) diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index d044a45bebce..93529595f706 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -6,6 +6,7 @@ import json import os + """ This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed diff --git a/tools/bin/record_obfuscator.py b/tools/bin/record_obfuscator.py index 4dff42a71a05..031010549fb1 100755 --- a/tools/bin/record_obfuscator.py +++ b/tools/bin/record_obfuscator.py @@ -7,6 +7,7 @@ import sys from typing import Any + # # record_obfuscator is a tiny script that: # 1. reads JSON lines from stdin diff --git a/tools/bin/update_intellij_venv.py b/tools/bin/update_intellij_venv.py index dfb259ee9255..d2474db6ac4b 100644 --- a/tools/bin/update_intellij_venv.py +++ b/tools/bin/update_intellij_venv.py @@ -8,6 +8,7 @@ import sys import xml.etree.ElementTree as ET + INTELLIJ_VERSION_FLAG = "-intellij-version" diff --git a/tools/schema_generator/schema_generator/infer_schemas.py b/tools/schema_generator/schema_generator/infer_schemas.py index 6e01353ce59c..bf58934480fb 100644 --- a/tools/schema_generator/schema_generator/infer_schemas.py +++ b/tools/schema_generator/schema_generator/infer_schemas.py @@ -27,10 +27,11 @@ import sys import genson.schema.strategies as strategies -from airbyte_cdk.models import AirbyteMessage, Type from genson import SchemaBuilder from genson.schema.strategies.object import Object +from airbyte_cdk.models import AirbyteMessage, Type + class NoRequiredObj(Object): """ diff --git a/tools/schema_generator/schema_generator/main.py b/tools/schema_generator/schema_generator/main.py index bd1cb14ae814..dd7f62bc75d0 100644 --- a/tools/schema_generator/schema_generator/main.py +++ b/tools/schema_generator/schema_generator/main.py @@ -10,7 +10,6 @@ def main(): - parser = argparse.ArgumentParser(description="Airbyte Schema Generator") if len(sys.argv) == 1: