From 03039390faa95e65d682403192a47ec770f4789c Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 18 Dec 2024 04:34:35 +0000 Subject: [PATCH] chore: auto-fix lint and format issues --- .../base_images/base_images/commands.py | 3 +- .../base_images/base_images/consts.py | 4 +- .../base_images/base_images/errors.py | 3 +- .../base_images/base_images/hacks.py | 1 + .../base_images/base_images/java/bases.py | 7 +- .../base_images/base_images/publish.py | 1 + .../base_images/base_images/python/bases.py | 1 + .../base_images/python/sanity_checks.py | 1 + .../base_images/base_images/root_images.py | 1 + .../base_images/base_images/sanity_checks.py | 1 + .../base_images/base_images/utils/docker.py | 2 +- .../base_images/version_registry.py | 2 + .../ci_credentials/ci_credentials/main.py | 1 + .../ci_credentials/ci_credentials/models.py | 1 + .../ci_credentials/secrets_manager.py | 1 + .../common_utils/common_utils/google_api.py | 1 + .../connector_ops/required_reviewer_checks.py | 2 + .../connector_ops/connector_ops/utils.py | 1 + .../src/connectors_insights/cli.py | 1 + .../src/connectors_insights/insights.py | 1 + .../src/connectors_insights/pylint.py | 1 + .../src/connectors_qa/checks/assets.py | 2 +- .../checks/documentation/documentation.py | 3 +- .../src/connectors_qa/checks/metadata.py | 3 +- .../src/connectors_qa/checks/packaging.py | 3 +- .../src/connectors_qa/checks/security.py | 3 +- .../src/connectors_qa/checks/testing.py | 3 +- .../connectors_qa/src/connectors_qa/cli.py | 3 +- .../connectors_qa/src/connectors_qa/models.py | 7 +- .../connectors_qa/src/connectors_qa/utils.py | 1 + .../integration_tests/test_documentation.py | 9 +- .../unit_tests/test_checks/test_assets.py | 1 - .../test_checks/test_documentation.py | 97 +-- .../unit_tests/test_checks/test_metadata.py | 1 + .../unit_tests/test_checks/test_packaging.py | 1 + .../unit_tests/test_checks/test_testing.py | 37 +- .../tests/unit_tests/test_models.py | 1 + .../connectors/erd/src/erd/dbml_assembler.py | 139 +++- .../connectors/erd/src/erd/erd_service.py | 40 +- .../connectors/erd/src/erd/relationships.py | 37 +- .../erd/tests/test_dbml_assembler.py | 5 +- .../erd/tests/test_relationships.py | 76 +- .../commons/backends/base_backend.py | 3 +- .../commons/backends/duckdb_backend.py | 1 + .../commons/backends/file_backend.py | 7 +- .../commons/connection_objects_retrieval.py | 1 + .../live_tests/commons/connector_runner.py | 1 + .../src/live_tests/commons/models.py | 21 +- .../live-tests/src/live_tests/conftest.py | 3 +- .../live_tests/regression_tests/test_check.py | 1 + .../regression_tests/test_discover.py | 1 + .../live_tests/regression_tests/test_read.py | 1 + .../live_tests/regression_tests/test_spec.py | 1 + .../live-tests/src/live_tests/report.py | 2 + .../live-tests/src/live_tests/stash_keys.py | 1 + .../live-tests/src/live_tests/utils.py | 5 +- .../live_tests/validation_tests/test_check.py | 1 + .../validation_tests/test_discover.py | 1 + .../live_tests/validation_tests/test_read.py | 1 + .../live_tests/validation_tests/test_spec.py | 1 + .../tests/backends/test_file_backend.py | 1 + .../tests/test_json_schema_helper.py | 3 +- .../lib/metadata_service/commands.py | 3 +- .../lib/metadata_service/gcs_upload.py | 5 +- .../validators/metadata_validator.py | 6 +- .../lib/tests/test_commands.py | 6 +- .../lib/tests/test_docker_hub.py | 1 + .../lib/tests/test_gcs_upload.py | 3 +- .../lib/tests/test_spec_cache.py | 9 +- .../lib/tests/test_transform.py | 1 + .../test_metadata_validators.py | 1 + .../orchestrator/assets/connector_metrics.py | 1 + .../assets/connector_test_report.py | 1 + .../orchestrator/assets/github.py | 1 + .../orchestrator/assets/metadata.py | 1 + .../orchestrator/assets/registry.py | 1 + .../orchestrator/assets/registry_entry.py | 1 + .../orchestrator/assets/registry_report.py | 1 + .../orchestrator/orchestrator/assets/slack.py | 1 + .../orchestrator/assets/specs_secrets_mask.py | 1 + .../orchestrator/orchestrator/config.py | 1 + .../fetcher/connector_cdk_version.py | 1 + .../orchestrator/orchestrator/hacks.py | 1 + .../jobs/connector_test_report.py | 1 + .../orchestrator/jobs/metadata.py | 1 + .../orchestrator/jobs/registry.py | 1 + .../orchestrator/logging/sentry.py | 1 + .../orchestrator/models/ci_report.py | 1 + .../file_managers/local_file_manager.py | 1 + .../orchestrator/sensors/github.py | 1 + .../orchestrator/templates/render.py | 1 + .../orchestrator/utils/dagster_helpers.py | 1 + .../orchestrator/utils/object_helpers.py | 1 + .../connectors/build_image/commands.py | 1 + .../connectors/build_image/steps/common.py | 1 + .../build_image/steps/java_connectors.py | 1 + .../steps/manifest_only_connectors.py | 3 +- .../build_image/steps/normalization.py | 1 + .../build_image/steps/python_connectors.py | 1 + .../connectors/bump_version/commands.py | 1 + .../airbyte_ci/connectors/commands.py | 1 + .../airbyte_ci/connectors/context.py | 1 + .../connectors/generate_erd/commands.py | 1 + .../connectors/generate_erd/pipeline.py | 3 +- .../airbyte_ci/connectors/list/commands.py | 3 +- .../migrate_to_base_image/commands.py | 1 + .../migrate_to_base_image/pipeline.py | 3 +- .../migrate_to_inline_schemas/commands.py | 1 + .../migrate_to_inline_schemas/pipeline.py | 1 + .../migrate_to_logging_logger/commands.py | 1 + .../migrate_to_logging_logger/pipeline.py | 1 + .../migrate_to_manifest_only/commands.py | 2 +- .../migrate_to_manifest_only/pipeline.py | 2 +- .../connectors/migrate_to_poetry/commands.py | 2 +- .../connectors/migrate_to_poetry/pipeline.py | 1 + .../airbyte_ci/connectors/pipeline.py | 2 + .../airbyte_ci/connectors/publish/commands.py | 1 + .../airbyte_ci/connectors/publish/context.py | 2 +- .../airbyte_ci/connectors/publish/pipeline.py | 9 +- .../connectors/pull_request/commands.py | 1 + .../airbyte_ci/connectors/reports.py | 20 +- .../airbyte_ci/connectors/test/commands.py | 1 + .../airbyte_ci/connectors/test/context.py | 3 +- .../airbyte_ci/connectors/test/pipeline.py | 2 +- .../connectors/test/steps/common.py | 9 +- .../connectors/test/steps/java_connectors.py | 3 +- .../test/steps/manifest_only_connectors.py | 1 + .../test/steps/python_connectors.py | 3 +- .../connectors/up_to_date/commands.py | 1 + .../connectors/up_to_date/pipeline.py | 2 + .../airbyte_ci/connectors/up_to_date/steps.py | 1 + .../connectors/upgrade_cdk/commands.py | 1 + .../connectors/upgrade_cdk/pipeline.py | 1 + .../pipelines/airbyte_ci/format/commands.py | 2 + .../pipelines/airbyte_ci/format/containers.py | 1 + .../airbyte_ci/format/format_command.py | 1 + .../pipelines/airbyte_ci/metadata/commands.py | 1 + .../pipelines/airbyte_ci/metadata/pipeline.py | 1 + .../pipelines/airbyte_ci/poetry/commands.py | 2 + .../airbyte_ci/poetry/publish/commands.py | 2 + .../pipelines/airbyte_ci/steps/base_image.py | 2 +- .../airbyte_ci/steps/bump_version.py | 1 + .../pipelines/airbyte_ci/steps/changelog.py | 2 +- .../pipelines/airbyte_ci/steps/docker.py | 1 + .../pipelines/airbyte_ci/steps/gradle.py | 3 +- .../airbyte_ci/steps/python_registry.py | 1 + .../pipelines/airbyte_ci/test/commands.py | 1 + .../pipelines/airbyte_ci/test/pipeline.py | 1 + .../pipelines/airbyte_ci/update/commands.py | 1 + .../pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/pipelines/cli/auto_update.py | 1 + .../pipelines/cli/click_decorators.py | 1 + .../pipelines/cli/dagger_pipeline_command.py | 2 + .../pipelines/pipelines/cli/dagger_run.py | 1 + .../pipelines/pipelines/cli/secrets.py | 1 + .../dagger/actions/connector/hooks.py | 1 + .../dagger/actions/connector/normalization.py | 1 + .../pipelines/dagger/actions/python/common.py | 1 + .../pipelines/dagger/actions/python/pipx.py | 1 + .../pipelines/dagger/actions/python/poetry.py | 1 + .../dagger/actions/remote_storage.py | 1 + .../pipelines/dagger/actions/secrets.py | 2 + .../pipelines/dagger/actions/system/docker.py | 9 +- .../pipelines/dagger/containers/git.py | 1 + .../dagger/containers/internal_tools.py | 1 + .../pipelines/dagger/containers/java.py | 1 + .../pipelines/dagger/containers/python.py | 1 + .../connectors/pipelines/pipelines/hacks.py | 2 + .../pipelines/pipelines/helpers/changelog.py | 1 + .../pipelines/pipelines/helpers/cli.py | 1 + .../pipelines/helpers/connectors/command.py | 1 + .../pipelines/helpers/connectors/modifed.py | 1 + .../pipelines/helpers/execution/run_steps.py | 2 +- .../pipelines/pipelines/helpers/gcs.py | 1 + .../pipelines/pipelines/helpers/git.py | 1 + .../pipelines/pipelines/helpers/github.py | 3 +- .../pipelines/helpers/sentry_utils.py | 1 + .../pipelines/pipelines/helpers/slack.py | 1 + .../pipelines/pipelines/helpers/utils.py | 1 + .../pipelines/pipelines/models/artifacts.py | 1 + .../models/contexts/click_pipeline_context.py | 4 +- .../models/contexts/pipeline_context.py | 8 +- .../pipelines/pipelines/models/reports.py | 12 +- .../pipelines/pipelines/models/steps.py | 7 +- .../connectors/pipelines/tests/conftest.py | 1 + .../tests/test_actions/test_environments.py | 8 +- .../connectors/pipelines/tests/test_bases.py | 1 + .../test_manifest_only_connectors.py | 1 + .../test_python_connectors.py | 1 + .../test_steps/test_common.py | 1 + .../pipelines/tests/test_changelog.py | 1 + .../tests/test_cli/test_click_decorators.py | 1 + .../test_groups/test_connectors.py | 1 + .../test_actions/test_python/test_common.py | 2 +- .../tests/test_format/test_commands.py | 1 + .../connectors/pipelines/tests/test_gradle.py | 3 +- .../test_execution/test_argument_parsing.py | 1 + .../test_execution/test_run_steps.py | 1 + .../pipelines/tests/test_helpers/test_pip.py | 1 + .../tests/test_helpers/test_utils.py | 2 +- .../test_click_pipeline_context.py | 1 + .../tests/test_poetry/test_poetry_publish.py | 1 + .../pipelines/tests/test_publish.py | 2 +- .../test_steps/test_simple_docker_step.py | 1 + .../tests/test_steps/test_version_check.py | 4 +- .../pipelines/tests/test_tests/test_common.py | 1 + .../test_tests/test_python_connectors.py | 1 + .../pipelines/tests/test_upgrade_java_cdk.py | 1 + .../connector_acceptance_test/base.py | 1 + .../connector_acceptance_test/config.py | 1 + .../connector_acceptance_test/conftest.py | 1 + .../connector_acceptance_test/plugin.py | 2 + .../utils/asserts.py | 4 +- .../utils/backward_compatibility.py | 5 +- .../utils/client_container_runner.py | 4 +- .../connector_acceptance_test/utils/common.py | 1 + .../utils/compare.py | 1 + .../utils/connector_runner.py | 5 +- .../utils/manifest_helper.py | 1 + .../unit_tests/test_asserts.py | 3 +- .../unit_tests/test_backward_compatibility.py | 3 +- .../unit_tests/test_connector_attributes.py | 81 ++- .../unit_tests/test_connector_runner.py | 5 +- .../unit_tests/test_core.py | 136 ++-- .../unit_tests/test_documentation.py | 127 ++-- .../unit_tests/test_global_fixtures.py | 3 +- .../unit_tests/test_incremental.py | 207 +++--- .../unit_tests/test_json_schema_helper.py | 5 +- .../unit_tests/test_plugin.py | 1 + .../unit_tests/test_spec.py | 552 +++++++-------- .../unit_tests/test_test_full_refresh.py | 6 +- .../unit_tests/test_utils.py | 3 +- .../integration_tests/acceptance.py | 1 + .../destination_amazon_sqs/destination.py | 4 +- .../integration_tests/integration_test.py | 3 +- .../connectors/destination-amazon-sqs/main.py | 1 + .../unit_tests/unit_test.py | 3 +- .../destination_astra/config.py | 3 +- .../destination_astra/destination.py | 1 + .../destination_astra/indexer.py | 2 + .../integration_tests/integration_test.py | 20 +- .../connectors/destination-astra/main.py | 1 + .../unit_tests/destination_test.py | 3 +- .../unit_tests/indexer_test.py | 15 +- .../destination_aws_datalake/aws.py | 4 +- .../destination_aws_datalake/destination.py | 5 +- .../destination_aws_datalake/stream_writer.py | 2 + .../integration_tests/integration_test.py | 22 +- .../destination-aws-datalake/main.py | 1 + .../unit_tests/stream_writer_test.py | 3 +- .../destination_chroma/config.py | 4 +- .../destination_chroma/destination.py | 3 +- .../destination_chroma/indexer.py | 3 +- .../connectors/destination-chroma/main.py | 1 + .../unit_tests/test_destination.py | 3 +- .../unit_tests/test_indexer.py | 4 +- .../destination_convex/client.py | 1 + .../destination_convex/config.py | 1 + .../destination_convex/destination.py | 1 + .../connectors/destination-convex/main.py | 1 + .../unit_tests/unit_test.py | 8 +- .../destination_cumulio/client.py | 1 + .../destination_cumulio/destination.py | 1 + .../integration_tests/integration_test.py | 5 +- .../connectors/destination-cumulio/main.py | 1 + .../unit_tests/test_client.py | 1 + .../unit_tests/test_destination.py | 3 +- .../unit_tests/test_writer.py | 3 +- .../destination_databend/destination.py | 2 +- .../integration_tests/integration_test.py | 5 +- .../connectors/destination-databend/main.py | 1 + .../unit_tests/test_databend_destination.py | 37 +- .../destination_duckdb/destination.py | 2 +- .../integration_tests/integration_test.py | 88 ++- .../connectors/destination-duckdb/main.py | 1 + .../unit_tests/destination_unit_tests.py | 4 + .../destination_firebolt/destination.py | 7 +- .../integration_tests/integration_test.py | 7 +- .../connectors/destination-firebolt/main.py | 1 + .../unit_tests/test_firebolt_destination.py | 7 +- .../integration_tests/integration_test.py | 7 +- .../connectors/destination-firestore/main.py | 1 + .../destination_google_sheets/buffer.py | 1 - .../destination_google_sheets/client.py | 5 +- .../destination_google_sheets/destination.py | 4 +- .../destination_google_sheets/helpers.py | 7 +- .../destination_google_sheets/writer.py | 3 +- .../integration_tests/test_buffer.py | 4 +- .../integration_tests/test_client.py | 1 + .../integration_tests/test_destination.py | 7 +- .../integration_tests/test_helpers.py | 4 +- .../integration_tests/test_spreadsheet.py | 1 + .../integration_tests/test_writer.py | 4 +- .../destination-google-sheets/main.py | 1 + .../destination_kvdb/destination.py | 1 - .../connectors/destination-kvdb/main.py | 1 + .../destination_meilisearch/run.py | 3 +- .../destination_milvus/config.py | 3 +- .../destination_milvus/destination.py | 1 + .../destination_milvus/indexer.py | 4 +- .../milvus_integration_test.py | 7 +- .../connectors/destination-milvus/main.py | 1 + .../unit_tests/destination_test.py | 3 +- .../unit_tests/indexer_test.py | 3 +- .../destination_motherduck/destination.py | 6 +- .../processors/duckdb.py | 12 +- .../processors/motherduck.py | 8 +- .../integration_tests/integration_test.py | 69 +- .../connectors/destination-motherduck/main.py | 1 + .../unit_tests/destination_unit_tests.py | 3 +- .../common/catalog/catalog_providers.py | 8 +- .../common/destinations/record_processor.py | 15 +- .../common/sql/sql_processor.py | 71 +- .../destination_pgvector/config.py | 1 - .../destination_pgvector/destination.py | 9 +- .../pgvector_processor.py | 29 +- .../integration_tests/integration_test.py | 5 +- .../unit_tests/destination_test.py | 1 + .../destination_pinecone/config.py | 3 +- .../destination_pinecone/destination.py | 1 + .../destination_pinecone/indexer.py | 6 +- .../pinecone_integration_test.py | 73 +- .../connectors/destination-pinecone/main.py | 1 + .../destination-pinecone/test_pinecone.py | 1 + .../unit_tests/destination_test.py | 3 +- .../unit_tests/pinecone_indexer_test.py | 32 +- .../destination_qdrant/config.py | 3 +- .../destination_qdrant/destination.py | 3 +- .../destination_qdrant/indexer.py | 8 +- .../connectors/destination-qdrant/main.py | 1 + .../unit_tests/test_destination.py | 3 +- .../unit_tests/test_indexer.py | 5 +- .../destination_rabbitmq/destination.py | 6 +- .../integration_tests/integration_test.py | 4 +- .../connectors/destination-rabbitmq/main.py | 1 + .../unit_tests/unit_test.py | 6 +- .../destination_sftp_json/destination.py | 1 - .../integration_tests/integration_test.py | 5 +- .../connectors/destination-sftp-json/main.py | 1 + .../connectors/destination-sftp-json/setup.py | 1 + .../common/sql/sql_processor.py | 11 +- .../common/state/state_writers.py | 1 - .../destination_snowflake_cortex/globals.py | 1 - .../integration_tests/integration_test.py | 4 +- .../unit_tests/destination_test.py | 2 +- .../destination_sqlite/destination.py | 19 +- .../integration_tests/integration_test.py | 3 +- .../connectors/destination-sqlite/main.py | 1 + .../destination_timeplus/destination.py | 4 +- .../integration_tests/integration_test.py | 3 +- .../connectors/destination-timeplus/main.py | 1 + .../destination_typesense/destination.py | 3 +- .../destination_typesense/writer.py | 3 +- .../integration_tests/integration_test.py | 5 +- .../connectors/destination-typesense/main.py | 1 + .../destination_vectara/client.py | 4 +- .../destination_vectara/config.py | 3 +- .../destination_vectara/destination.py | 1 - .../destination_vectara/writer.py | 3 +- .../integration_tests/integration_test.py | 6 +- .../connectors/destination-vectara/main.py | 1 + .../destination_weaviate/config.py | 3 +- .../destination_weaviate/indexer.py | 1 + .../integration_tests/integration_test.py | 8 +- .../connectors/destination-weaviate/main.py | 1 + .../unit_tests/destination_test.py | 3 +- .../unit_tests/indexer_test.py | 5 +- .../destination_xata/destination.py | 6 +- .../integration_tests/integration_test.py | 5 +- .../connectors/destination-xata/main.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-adjust/main.py | 1 + .../source-adjust/source_adjust/components.py | 1 - .../source-adjust/source_adjust/model.py | 1 + .../source-adjust/source_adjust/source.py | 1 + .../source-adjust/unit_tests/conftest.py | 17 +- .../unit_tests/test_incremental_streams.py | 3 +- .../source-adjust/unit_tests/test_streams.py | 13 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-airtable/main.py | 1 + .../airtable_backoff_strategy.py | 1 + .../source_airtable/airtable_error_handler.py | 1 + .../source_airtable/airtable_error_mapping.py | 1 + .../source-airtable/source_airtable/auth.py | 1 + .../source_airtable/schema_helpers.py | 2 +- .../source-airtable/source_airtable/source.py | 1 - .../source_airtable/streams.py | 3 +- .../source-airtable/unit_tests/conftest.py | 3 +- .../test_airtable_backoff_strategy.py | 8 +- .../unit_tests/test_airtable_error_handler.py | 22 +- .../unit_tests/test_authenticator.py | 4 +- .../source-airtable/unit_tests/test_source.py | 3 +- .../unit_tests/test_streams.py | 2 - .../integration_tests/acceptance.py | 1 + .../connectors/source-alpha-vantage/main.py | 1 + .../object_dpath_extractor.py | 1 + .../source_alpha_vantage/run.py | 3 +- .../source_alpha_vantage/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-amazon-ads/main.py | 1 + .../source_amazon_ads/config_migrations.py | 1 + .../source_amazon_ads/run.py | 3 +- .../source_amazon_ads/source.py | 2 + .../source_amazon_ads/spec.py | 3 +- .../source_amazon_ads/streams/common.py | 4 +- .../source_amazon_ads/streams/profiles.py | 1 + .../streams/report_streams/brands_report.py | 1 + .../streams/report_streams/display_report.py | 2 + .../streams/report_streams/products_report.py | 2 + .../streams/report_streams/report_streams.py | 3 +- .../source_amazon_ads/utils.py | 1 + .../attribution_report_request_builder.py | 1 + .../unit_tests/integrations/config.py | 1 + .../test_attribution_report_streams.py | 2 + .../integrations/test_report_streams.py | 6 +- .../integrations/test_sponsored_streams.py | 1 + .../unit_tests/integrations/utils.py | 6 +- .../unit_tests/test_report_streams.py | 6 +- .../source-amazon-ads/unit_tests/utils.py | 3 +- .../integration_tests/acceptance.py | 1 + .../source-amazon-seller-partner/main.py | 1 + .../source_amazon_seller_partner/auth.py | 1 + .../config_migrations.py | 1 + .../source_amazon_seller_partner/constants.py | 1 + .../source_amazon_seller_partner/source.py | 4 +- .../source_amazon_seller_partner/streams.py | 2 + .../source_amazon_seller_partner/utils.py | 1 + .../unit_tests/conftest.py | 6 +- .../unit_tests/integration/config.py | 1 + .../unit_tests/integration/pagination.py | 1 + .../integration/test_report_based_streams.py | 14 +- ...test_vendor_direct_fulfillment_shipping.py | 2 + .../integration/test_vendor_orders.py | 2 + .../unit_tests/integration/utils.py | 3 +- .../unit_tests/test_analytics_streams.py | 3 +- .../unit_tests/test_finance_streams.py | 14 +- .../unit_tests/test_migrations.py | 6 +- .../test_reports_streams_settlement_report.py | 4 +- .../unit_tests/test_source.py | 24 +- .../unit_tests/test_streams.py | 17 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-amazon-sqs/main.py | 1 + .../connectors/source-amazon-sqs/setup.py | 1 + .../source_amazon_sqs/source.py | 3 +- .../source-amazon-sqs/unit_tests/unit_test.py | 5 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/integration_test.py | 3 +- .../connectors/source-amplitude/main.py | 1 + .../source_amplitude/components.py | 2 + .../source-amplitude/source_amplitude/run.py | 3 +- .../source_amplitude/source.py | 4 +- .../source_amplitude/streams.py | 2 + .../unit_tests/test_custom_extractors.py | 10 +- .../source-apify-dataset/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-appsflyer/main.py | 1 + .../source_appsflyer/source.py | 3 +- .../unit_tests/test_incremental_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-asana/main.py | 1 + .../source-asana/source_asana/components.py | 3 +- .../source_asana/config_migration.py | 1 + .../source-asana/source_asana/run.py | 5 +- .../source-asana/source_asana/source.py | 1 + .../unit_tests/test_config_migrations.py | 11 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-avni/main.py | 4 +- .../source-avni/source_avni/components.py | 3 +- .../connectors/source-avni/source_avni/run.py | 3 +- .../source-avni/source_avni/source.py | 1 + .../source-avni/unit_tests/test_components.py | 25 +- .../source-aws-cloudtrail/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../build_customization.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/conftest.py | 4 +- .../integration_tests/integration_test.py | 3 +- .../source-azure-blob-storage/main.py | 1 + .../config_migrations.py | 7 +- .../source_azure_blob_storage/spec.py | 3 +- .../stream_reader.py | 9 +- .../unit_tests/test_authenticator.py | 13 +- .../unit_tests/test_config_migration.py | 3 +- .../unit_tests/test_stream_reader.py | 30 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-azure-table/main.py | 1 + .../unit_tests/test_azure_table.py | 22 +- .../unit_tests/test_source.py | 3 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-bamboo-hr/components.py | 1 - .../integration_tests/acceptance.py | 1 + .../source-bigcommerce/components.py | 2 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-bing-ads/main.py | 1 + .../source_bing_ads/base_streams.py | 7 +- .../source_bing_ads/bulk_streams.py | 4 +- .../source-bing-ads/source_bing_ads/client.py | 6 +- .../source_bing_ads/report_streams.py | 13 +- .../unit_tests/integrations/base_test.py | 11 +- .../unit_tests/integrations/config_builder.py | 1 + .../integrations/suds_response_mock.py | 1 + .../integrations/test_accounts_stream.py | 9 +- .../test_app_install_ad_labels_stream.py | 23 +- .../test_app_install_ads_stream.py | 21 +- .../integrations/test_budget_stream.py | 5 +- .../integrations/test_hourly_reports.py | 1 + .../integrations/test_report_stream.py | 5 +- .../unit_tests/test_bulk_streams.py | 54 +- .../source-bing-ads/unit_tests/test_client.py | 3 +- .../unit_tests/test_reports.py | 60 +- .../source-bing-ads/unit_tests/test_source.py | 10 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-braintree/main.py | 1 + .../source_braintree/schemas/common.py | 3 +- .../source_braintree/source.py | 8 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-braze/main.py | 1 + .../connectors/source-braze/setup.py | 1 + .../source-braze/source_braze/components.py | 1 + .../source-braze/source_braze/run.py | 3 +- .../source-braze/source_braze/source.py | 1 + .../source_braze/transformations.py | 1 + .../test_datetime_incremental_sync.py | 3 +- .../unit_tests/test_transformations.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-cart/main.py | 1 + .../source-cart/source_cart/source.py | 3 +- .../source-cart/source_cart/streams.py | 2 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-chargebee/main.py | 1 + .../source_chargebee/components.py | 1 - .../source-chargebee/source_chargebee/run.py | 5 +- .../source_chargebee/source.py | 1 + .../unit_tests/integration/config.py | 4 +- .../unit_tests/integration/pagination.py | 2 +- .../unit_tests/integration/request_builder.py | 17 +- .../integration/response_builder.py | 3 +- .../unit_tests/integration/test_addon.py | 57 +- .../unit_tests/integration/test_coupon.py | 41 +- .../unit_tests/integration/test_customer.py | 56 +- .../unit_tests/integration/test_event.py | 35 +- .../integration/test_hosted_page.py | 49 +- .../unit_tests/integration/test_plan.py | 53 +- .../integration/test_site_migration_detail.py | 60 +- .../integration/test_subscription.py | 55 +- ...est_subscription_with_scheduled_changes.py | 82 ++- .../integration/test_virtual_bank_account.py | 55 +- .../unit_tests/test_component.py | 32 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-close-com/main.py | 1 + .../datetime_incremental_sync.py | 1 + .../source_close_com/source.py | 1 + .../source_close_com/source_lc.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-commcare/main.py | 1 + .../source-commcare/source_commcare/source.py | 9 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-commercetools/main.py | 1 + .../source_commercetools/components.py | 2 + .../source_commercetools/run.py | 3 +- .../source_commercetools/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-convex/main.py | 1 + .../source-convex/source_convex/source.py | 2 + .../unit_tests/test_incremental_streams.py | 3 +- .../source-convex/unit_tests/test_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-declarative-manifest/main.py | 1 + .../source_declarative_manifest/run.py | 3 +- .../test_source_declarative_local_manifest.py | 13 +- ...test_source_declarative_remote_manifest.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-facebook-marketing/main.py | 1 + .../source_facebook_marketing/api.py | 2 + .../config_migrations.py | 1 + .../source_facebook_marketing/source.py | 2 + .../source_facebook_marketing/spec.py | 6 +- .../streams/async_job.py | 2 + .../streams/async_job_manager.py | 1 + .../streams/base_insight_streams.py | 6 +- .../streams/base_streams.py | 6 +- .../streams/common.py | 4 +- .../streams/streams.py | 4 +- .../source_facebook_marketing/utils.py | 1 + .../unit_tests/conftest.py | 1 + .../unit_tests/integration/config.py | 1 + .../unit_tests/integration/pagination.py | 1 + .../test_ads_insights_action_product_id.py | 236 ++++--- .../unit_tests/integration/test_videos.py | 16 +- .../unit_tests/integration/utils.py | 7 +- .../unit_tests/test_api.py | 1 + .../unit_tests/test_base_insight_streams.py | 244 ++++--- .../unit_tests/test_client.py | 10 +- .../unit_tests/test_config_migrations.py | 13 +- .../unit_tests/test_errors.py | 108 +-- .../unit_tests/test_source.py | 7 +- .../unit_tests/test_utils.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-facebook-pages/main.py | 1 + .../source_facebook_pages/components.py | 3 +- .../source_facebook_pages/run.py | 3 +- .../source_facebook_pages/source.py | 1 + .../source-facebook-pages/tools/schema_gen.py | 5 +- .../test_custom_field_transformation.py | 7 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-faker/main.py | 1 + .../source_faker/purchase_generator.py | 3 +- .../source-faker/source_faker/source.py | 1 + .../source_faker/user_generator.py | 3 +- .../source-faker/unit_tests/unit_test.py | 3 +- .../connectors/source-fastbill/components.py | 1 - .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-fauna/main.py | 1 + .../connectors/source-fauna/setup.py | 1 + .../source-fauna/source_fauna/source.py | 11 +- .../source-fauna/unit_tests/check_test.py | 3 +- .../source-fauna/unit_tests/database_test.py | 7 +- .../source-fauna/unit_tests/discover_test.py | 3 +- .../unit_tests/incremental_test.py | 10 +- .../source-file/build_customization.py | 1 + .../integration_tests/acceptance.py | 1 + .../client_storage_providers_test.py | 1 + .../source-file/integration_tests/conftest.py | 1 + .../integration_tests/file_formats_test.py | 4 +- .../connectors/source-file/main.py | 1 + .../source-file/source_file/client.py | 8 +- .../source-file/source_file/utils.py | 1 + .../source-file/unit_tests/test_client.py | 12 +- .../unit_tests/test_nested_json_schema.py | 1 + .../source-file/unit_tests/test_source.py | 4 +- .../integration_tests/acceptance.py | 1 + .../source-firebase-realtime-database/main.py | 1 + .../unit_tests/unit_test.py | 1 - .../integration_tests/acceptance.py | 1 + .../integration_tests/integration_test.py | 7 +- .../connectors/source-firebolt/main.py | 1 + .../source-firebolt/source_firebolt/source.py | 1 + .../unit_tests/test_firebolt_source.py | 13 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-freshcaller/main.py | 1 + .../source_freshcaller/run.py | 3 +- .../source_freshcaller/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-freshdesk/main.py | 1 + .../source_freshdesk/components.py | 5 +- .../unit_tests/test_300_page.py | 3 +- .../unit_tests/test_incremental_sync.py | 3 +- .../unit_tests/test_pagination_strategy.py | 1 - .../unit_tests/test_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-gcs/build_customization.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-gcs/integration_tests/conftest.py | 2 +- .../integration_tests/integration_test.py | 3 +- .../connectors/source-gcs/main.py | 1 + .../source-gcs/source_gcs/config.py | 3 +- .../source-gcs/source_gcs/helpers.py | 3 +- .../connectors/source-gcs/source_gcs/run.py | 3 +- .../connectors/source-gcs/source_gcs/spec.py | 3 +- .../source-gcs/source_gcs/stream_reader.py | 7 +- .../source-gcs/source_gcs/zip_helper.py | 2 + .../source-gcs/unit_tests/conftest.py | 6 +- .../source-gcs/unit_tests/test_config.py | 1 - .../unit_tests/test_config_migrations.py | 18 +- .../source-gcs/unit_tests/test_cursor.py | 8 +- .../source-gcs/unit_tests/test_run.py | 3 +- .../source-gcs/unit_tests/test_source.py | 5 +- .../source-gcs/unit_tests/test_stream.py | 17 +- .../unit_tests/test_stream_reader.py | 10 +- .../source-gcs/unit_tests/test_zip_helper.py | 3 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-genesys/main.py | 1 + .../source_genesys/authenicator.py | 1 + .../source-genesys/source_genesys/source.py | 3 +- .../integration_tests/acceptance.py | 1 + .../source-github/fixtures/github.py | 1 + .../connectors/source-github/fixtures/main.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-github/main.py | 1 + .../source_github/backoff_strategies.py | 1 + .../source_github/config_migrations.py | 9 +- .../source_github/errors_handlers.py | 2 + .../source_github/github_schema.py | 1 + .../source-github/source_github/graphql.py | 2 +- .../source-github/source_github/source.py | 1 - .../source-github/source_github/streams.py | 12 +- .../source-github/source_github/utils.py | 12 +- .../source-github/unit_tests/conftest.py | 1 + .../unit_tests/integration/test_assignees.py | 23 +- .../unit_tests/integration/test_events.py | 4 +- .../test_migrations/test_config_migrations.py | 6 +- .../test_multiple_token_authenticator.py | 5 +- .../source-github/unit_tests/test_source.py | 5 +- .../source-github/unit_tests/test_stream.py | 71 +- .../source-github/unit_tests/utils.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-gitlab/main.py | 1 + .../source_gitlab/config_migrations.py | 7 +- .../source-gitlab/unit_tests/conftest.py | 4 +- .../unit_tests/test_config_migrations.py | 1 + .../unit_tests/test_partition_routers.py | 7 +- .../source-gitlab/unit_tests/test_source.py | 11 +- .../source-gitlab/unit_tests/test_streams.py | 9 +- .../source-gitlab/unit_tests/test_utils.py | 2 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-gnews/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../unit_tests/test_request_with_filter.py | 32 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/integration_test.py | 3 +- .../connectors/source-google-ads/main.py | 1 + .../source_google_ads/config_migrations.py | 1 + .../source_google_ads/custom_query_stream.py | 1 + .../source_google_ads/google_ads.py | 6 +- .../source_google_ads/source.py | 3 +- .../source_google_ads/streams.py | 9 +- .../source_google_ads/utils.py | 6 +- .../source-google-ads/unit_tests/conftest.py | 1 + .../unit_tests/test_config_migrations.py | 6 +- .../unit_tests/test_errors.py | 8 +- .../unit_tests/test_google_ads.py | 30 +- .../test_incremental_events_streams.py | 41 +- .../unit_tests/test_source.py | 5 +- .../unit_tests/test_streams.py | 12 +- .../unit_tests/test_utils.py | 3 +- .../integration_tests/acceptance.py | 1 + .../source-google-analytics-data-api/main.py | 1 + .../api_quota.py | 3 +- .../authenticator.py | 1 + .../config_migrations.py | 2 + ...alytics_data_api_metadata_error_mapping.py | 1 + .../source.py | 4 +- .../source_google_analytics_data_api/utils.py | 2 + .../unit_tests/conftest.py | 1 + .../unit_tests/test_api_quota.py | 6 +- .../unit_tests/test_migration.py | 3 +- .../test_config_migration_cohortspec.py | 6 +- .../test_migrations/test_config_migrations.py | 6 +- .../unit_tests/test_source.py | 65 +- .../unit_tests/test_streams.py | 71 +- .../integration_tests/acceptance.py | 1 + .../main.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-google-analytics-v4/main.py | 1 + .../custom_reports_validator.py | 3 +- .../source_google_analytics_v4/source.py | 3 +- .../unit_tests/conftest.py | 1 + .../test_custom_reports_validator.py | 3 +- .../unit_tests/unit_test.py | 4 +- .../integration_tests/acceptance.py | 1 + .../source-google-directory/main.py | 1 + .../source_google_directory/api.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-google-drive/main.py | 1 + .../source_google_drive/spec.py | 3 +- .../source_google_drive/stream_reader.py | 8 +- .../unit_tests/test_reader.py | 7 +- .../unit_tests/test_utils.py | 4 +- .../integration_tests/acceptance.py | 1 + .../credentials/get_authentication_url.py | 1 + .../credentials/get_refresh_token.py | 1 + .../credentials/setup.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-google-search-console/main.py | 1 + .../config_migrations.py | 1 + .../service_account_authenticator.py | 2 + .../source_google_search_console/source.py | 2 + .../source_google_search_console/streams.py | 4 +- .../test_migrations/test_config_migrations.py | 6 +- .../unit_tests/unit_test.py | 10 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-google-sheets/main.py | 1 + .../source_google_sheets/client.py | 1 + .../source_google_sheets/helpers.py | 4 +- .../source_google_sheets/source.py | 8 +- .../source_google_sheets/utils.py | 1 + .../unit_tests/conftest.py | 8 +- .../integration/custom_http_mocker.py | 26 +- .../unit_tests/integration/request_builder.py | 5 +- .../integration/test_credentials.py | 5 +- .../unit_tests/integration/test_source.py | 220 +++--- .../unit_tests/test_helpers.py | 8 +- .../unit_tests/test_stream.py | 50 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-greenhouse/main.py | 1 + .../source_greenhouse/source.py | 1 + .../source-greenhouse/unit_tests/conftest.py | 12 +- .../unit_tests/test_components.py | 72 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-gridly/main.py | 1 + .../source-gridly/source_gridly/helpers.py | 1 + .../source-gridly/source_gridly/source.py | 1 + .../source-gridly/unit_tests/test_source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-hardcoded-records/main.py | 1 + .../source_hardcoded_records/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-harness/main.py | 1 + .../source-harness/source_harness/run.py | 3 +- .../source-harness/source_harness/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/test_associations.py | 3 +- .../connectors/source-hubspot/main.py | 1 + .../source-hubspot/source_hubspot/errors.py | 3 +- .../source-hubspot/source_hubspot/source.py | 4 +- .../source-hubspot/source_hubspot/streams.py | 8 +- .../source-hubspot/unit_tests/conftest.py | 10 +- .../unit_tests/integrations/config_builder.py | 4 +- .../integrations/request_builders/api.py | 3 +- .../integrations/request_builders/streams.py | 38 +- .../contact_response_builder.py | 2 +- .../integrations/response_builder/helpers.py | 2 +- .../response_builder/pagination.py | 11 +- .../integrations/response_builder/streams.py | 2 +- .../test_contacts_form_submissions.py | 154 ++-- .../test_contacts_list_memberships.py | 87 ++- .../test_contacts_merged_audit.py | 131 ++-- .../integrations/test_engagements_calls.py | 40 +- .../unit_tests/integrations/test_leads.py | 39 +- .../integrations/test_owners_archived.py | 12 +- .../test_web_analytics_streams.py | 138 ++-- .../unit_tests/test_components.py | 36 +- .../unit_tests/test_field_type_converting.py | 1 - .../source-hubspot/unit_tests/test_source.py | 22 +- .../unit_tests/test_split_properties.py | 1 + .../source-hubspot/unit_tests/test_streams.py | 70 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/conftest.py | 1 + .../integration_tests/test_streams.py | 15 +- .../connectors/source-instagram/main.py | 1 + .../source-instagram/source_instagram/api.py | 4 +- .../source_instagram/common.py | 1 + .../source_instagram/components.py | 2 + .../source_instagram/source.py | 2 + .../source_instagram/streams.py | 3 +- .../source-instagram/unit_tests/conftest.py | 10 +- .../unit_tests/integration/config.py | 1 + .../unit_tests/integration/pagination.py | 1 + .../unit_tests/integration/request_builder.py | 5 +- .../integration/response_builder.py | 19 +- .../unit_tests/integration/test_api.py | 12 +- .../unit_tests/integration/test_media.py | 50 +- .../integration/test_media_insights.py | 131 ++-- .../unit_tests/integration/test_stories.py | 10 +- .../integration/test_story_insights.py | 49 +- .../test_user_lifetime_insights.py | 12 +- .../unit_tests/integration/test_users.py | 14 +- .../unit_tests/integration/utils.py | 3 +- .../source-instagram/unit_tests/records.py | 659 +++++------------- .../unit_tests/test_source.py | 22 +- .../unit_tests/test_streams.py | 6 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-instatus/main.py | 1 + .../source_instatus/components.py | 1 + .../source-instatus/source_instatus/run.py | 3 +- .../source-instatus/source_instatus/source.py | 1 + .../unit_tests/test_components.py | 3 +- .../connectors/source-intercom/components.py | 3 +- .../integration_tests/acceptance.py | 1 + .../unit_tests/test_components.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-iterable/main.py | 1 + .../source_iterable/components.py | 1 + .../source-iterable/source_iterable/source.py | 2 + .../source_iterable/streams.py | 13 +- .../source-iterable/unit_tests/conftest.py | 1 + .../test_export_adjustable_range.py | 4 +- .../unit_tests/test_exports_stream.py | 3 +- .../unit_tests/test_extractors.py | 3 +- .../unit_tests/test_slice_generator.py | 1 + .../unit_tests/test_stream_events.py | 3 +- .../unit_tests/test_streams.py | 8 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-jina-ai-reader/main.py | 1 + .../source_jina_ai_reader/config_migration.py | 1 + .../source_jina_ai_reader/source.py | 1 + .../unit_tests/test_config_migrations.py | 3 +- .../integration_tests/acceptance.py | 1 + .../fixtures/data_generator/generator.py | 3 +- .../fixtures/data_generator/streams.py | 3 +- .../connectors/source-jira/main.py | 1 + .../source_jira/components/extractors.py | 3 +- .../connectors/source-jira/source_jira/run.py | 3 +- .../source-jira/source_jira/source.py | 5 +- .../source-jira/source_jira/streams.py | 5 +- .../source_jira/type_transfromer.py | 1 + .../source-jira/source_jira/utils.py | 1 + .../source-jira/unit_tests/conftest.py | 1 + .../unit_tests/integration/test_issues.py | 29 +- .../source-jira/unit_tests/test_components.py | 3 +- .../source-jira/unit_tests/test_source.py | 3 +- .../source-jira/unit_tests/test_streams.py | 127 ++-- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-klaviyo/main.py | 1 + .../source_klaviyo/availability_strategy.py | 3 +- .../components/included_fields_extractor.py | 1 + .../components/klaviyo_error_handler.py | 3 +- .../source-klaviyo/source_klaviyo/run.py | 3 +- .../source-klaviyo/source_klaviyo/streams.py | 5 +- .../unit_tests/integration/config.py | 2 +- .../unit_tests/integration/test_profiles.py | 14 +- .../unit_tests/test_included_extractor.py | 22 +- .../test_per_partition_state_migration.py | 24 +- .../source-klaviyo/unit_tests/test_source.py | 14 +- .../source-klaviyo/unit_tests/test_streams.py | 73 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-kyriba/main.py | 1 + .../source-kyriba/source_kyriba/source.py | 1 + .../unit_tests/test_bank_balances_stream.py | 7 +- .../unit_tests/test_cash_flows.py | 3 +- .../unit_tests/test_incremental_streams.py | 3 +- .../source-kyriba/unit_tests/test_source.py | 2 + .../source-kyriba/unit_tests/test_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-kyve/main.py | 1 + .../source-kyve/source_kyve/source.py | 3 +- .../source-kyve/source_kyve/stream.py | 2 + .../unit_tests/test_incremental_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-linkedin-ads/main.py | 1 + .../source_linkedin_ads/components.py | 3 +- .../source_linkedin_ads/config_migrations.py | 1 + .../source_linkedin_ads/source.py | 1 + .../source_linkedin_ads/utils.py | 1 + .../unit_tests/conftest.py | 1 + .../unit_tests/test_components.py | 9 +- .../unit_tests/test_source.py | 44 +- .../unit_tests/test_streams.py | 9 +- .../samples/test_data_for_tranform.py | 7 +- .../utils_tests/test_update_specific_key.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-linnworks/main.py | 1 + .../source_linnworks/source.py | 2 +- .../source_linnworks/streams.py | 5 +- .../unit_tests/test_incremental_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-looker/main.py | 1 + .../source-looker/source_looker/components.py | 3 +- .../source-looker/source_looker/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-mailchimp/main.py | 1 + .../source_mailchimp/components.py | 1 + .../source_mailchimp/config_migrations.py | 2 + .../integration/test_automations.py | 4 +- ...mponent_custom_email_activity_extractor.py | 3 +- .../test_config_datacenter_migration.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-marketo/main.py | 1 + .../source-marketo/source_marketo/source.py | 1 + .../source-marketo/source_marketo/utils.py | 1 + .../source-marketo/unit_tests/conftest.py | 13 +- .../source-marketo/unit_tests/test_source.py | 42 +- .../source-marketo/unit_tests/test_utils.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-microsoft-dataverse/main.py | 1 + .../source_microsoft_dataverse/dataverse.py | 3 +- .../source_microsoft_dataverse/streams.py | 3 +- .../unit_tests/test_source.py | 3 +- .../integration_tests/acceptance.py | 1 + .../source-microsoft-onedrive/main.py | 1 + .../source_microsoft_onedrive/spec.py | 3 +- .../stream_reader.py | 7 +- .../integration_tests/acceptance.py | 1 + .../source-microsoft-sharepoint/main.py | 1 + .../source_microsoft_sharepoint/spec.py | 3 +- .../stream_reader.py | 5 +- .../source_microsoft_sharepoint/utils.py | 1 + .../unit_tests/test_stream_reader.py | 10 +- .../unit_tests/test_utils.py | 10 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-mixpanel/main.py | 1 + .../source_mixpanel/backoff_strategy.py | 1 + .../source_mixpanel/components.py | 3 +- .../source_mixpanel/config_migrations.py | 1 + .../errors_handlers/base_errors_handler.py | 1 + .../errors_handlers/export_errors_handler.py | 1 + .../source-mixpanel/source_mixpanel/source.py | 1 + .../source_mixpanel/streams/base.py | 5 +- .../source_mixpanel/streams/export.py | 1 + .../source-mixpanel/unit_tests/conftest.py | 2 + .../unit_tests/test_migration.py | 4 +- .../test_property_transformation.py | 4 +- .../source-mixpanel/unit_tests/test_source.py | 19 +- .../unit_tests/test_streams.py | 373 +++++----- .../source-mixpanel/unit_tests/unit_test.py | 1 - .../source-mixpanel/unit_tests/utils.py | 4 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-monday/main.py | 1 + .../source-monday/source_monday/components.py | 2 + .../source-monday/source_monday/extractor.py | 2 + .../source_monday/item_pagination_strategy.py | 1 + .../source-monday/source_monday/source.py | 1 + .../monday_requests/base_requests_builder.py | 7 +- .../error_response_builder.py | 1 - .../unit_tests/integrations/utils.py | 3 +- .../unit_tests/test_components.py | 9 +- .../unit_tests/test_graphql_requester.py | 17 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-my-hours/components.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/seed/hook.py | 58 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 2 +- .../connectors/source-netsuite/main.py | 1 + .../connectors/source-netsuite/setup.py | 1 + .../source-netsuite/source_netsuite/source.py | 5 +- .../source_netsuite/streams.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-notion/main.py | 1 + .../source-notion/source_notion/streams.py | 8 +- .../unit_tests/test_components.py | 136 ++-- .../unit_tests/test_python_streams.py | 167 ++++- .../source-notion/unit_tests/test_source.py | 6 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-okta/main.py | 1 + .../source-okta/source_okta/components.py | 1 + .../source_okta/config_migration.py | 1 + .../source-okta/source_okta/source.py | 1 + .../source-okta/unit_tests/test_migration.py | 6 +- .../source-okta/unit_tests/test_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-orb/components.py | 2 - .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-outbrain-amplify/main.py | 1 + .../source_outbrain_amplify/auth.py | 3 +- .../source_outbrain_amplify/source.py | 21 +- .../unit_tests/test_incremental_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-outreach/main.py | 1 + .../source_outreach/components.py | 1 + .../source-outreach/source_outreach/run.py | 3 +- .../source-outreach/source_outreach/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-pagerduty/main.py | 1 + .../source-pagerduty/source_pagerduty/run.py | 3 +- .../source_pagerduty/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-partnerstack/main.py | 1 + .../source_partnerstack/run.py | 3 +- .../source_partnerstack/source.py | 1 + .../bin/disputes_generator.py | 1 - .../bin/fixture_helper.py | 1 + .../bin/payments_generator.py | 1 - .../bin/paypal_transaction_generator.py | 3 +- .../integration_tests/acceptance.py | 1 + .../source-paypal-transaction/main.py | 1 + .../source_paypal_transaction/components.py | 2 + .../source_paypal_transaction/run.py | 3 +- .../source_paypal_transaction/source.py | 2 + .../unit_tests/auth_components_test.py | 48 +- .../unit_tests/conftest.py | 39 +- .../unit_tests/pagination_cursor.py | 44 +- .../unit_tests/pagination_increment.py | 24 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-pinterest/main.py | 1 + .../source_pinterest/components/auth.py | 2 + .../source_pinterest/python_stream_auth.py | 1 + .../source_pinterest/reports/reports.py | 3 +- .../source_pinterest/source.py | 2 + .../source_pinterest/streams.py | 2 + .../source-pinterest/unit_tests/conftest.py | 3 +- .../source-pinterest/unit_tests/test_auth.py | 11 +- .../unit_tests/test_incremental_streams.py | 8 +- .../unit_tests/test_reports.py | 53 +- .../unit_tests/test_source.py | 6 +- .../unit_tests/test_streams.py | 17 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-pipedrive/main.py | 1 + .../source_pipedrive/extractor.py | 1 + .../source-pipedrive/source_pipedrive/run.py | 5 +- .../source_pipedrive/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-pocket/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/seed/hook.py | 75 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-posthog/main.py | 1 + .../source-posthog/source_posthog/source.py | 1 + .../unit_tests/test_components.py | 4 +- .../source-posthog/unit_tests/unit_test.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-prestashop/main.py | 1 + .../source_prestashop/components.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-public-apis/main.py | 1 + .../source_public_apis/components.py | 2 +- .../source_public_apis/run.py | 3 +- .../source_public_apis/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-python-http-tutorial/main.py | 1 + .../source-python-http-tutorial/setup.py | 1 + .../source_python_http_tutorial/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-qualaroo/components.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-quickbooks/main.py | 1 + .../source_quickbooks/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-railz/main.py | 1 + .../source-railz/source_railz/components.py | 3 +- .../source-railz/source_railz/run.py | 3 +- .../source-railz/source_railz/source.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-recharge/main.py | 1 + .../components/recharge_error_handler.py | 4 +- .../source-recharge/source_recharge/source.py | 1 + .../source_recharge/streams.py | 1 + .../unit_tests/integration/config.py | 1 + .../unit_tests/integration/pagination.py | 1 + .../unit_tests/integration/request_builder.py | 2 +- .../integration/streams/test_collections.py | 2 + .../integration/streams/test_discounts.py | 3 +- .../integration/streams/test_events.py | 3 +- .../integration/streams/test_onetimes.py | 3 +- .../integration/streams/test_shop.py | 2 + .../unit_tests/integration/utils.py | 3 +- .../unit_tests/test_streams.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-recurly/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-rki-covid/main.py | 1 + .../source_rki_covid/source.py | 3 +- .../unit_tests/test_incremental_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-rss/main.py | 1 + .../source-rss/source_rss/components.py | 3 +- .../connectors/source-rss/source_rss/run.py | 3 +- .../source-rss/source_rss/source.py | 1 + .../source-s3/integration_tests/acceptance.py | 1 + .../integration_tests/test_acceptance.py | 8 +- .../connectors/source-s3/main.py | 1 + .../source-s3/scripts/fetch_test_secrets.py | 1 + .../source-s3/scripts/rotate_creds.py | 1 + .../source_s3/source_files_abstract/source.py | 4 +- .../source_s3/source_files_abstract/spec.py | 1 + .../connectors/source-s3/source_s3/stream.py | 7 +- .../connectors/source-s3/source_s3/utils.py | 1 + .../source-s3/source_s3/v4/config.py | 5 +- .../source-s3/source_s3/v4/cursor.py | 1 + .../source_s3/v4/legacy_config_transformer.py | 1 + .../source-s3/source_s3/v4/source.py | 2 + .../source-s3/source_s3/v4/stream_reader.py | 12 +- .../source-s3/source_s3/v4/zip_reader.py | 4 +- .../source-s3/unit_tests/v4/test_config.py | 1 + .../source-s3/unit_tests/v4/test_cursor.py | 3 +- .../source-s3/unit_tests/v4/test_source.py | 1 + .../unit_tests/v4/test_stream_reader.py | 89 +-- .../integration_tests/acceptance.py | 1 + .../integration_tests/bulk_error_test.py | 4 +- .../integration_tests/integration_test.py | 12 +- .../integration_tests/state_migration.py | 3 +- .../connectors/source-salesforce/main.py | 1 + .../source_salesforce/api.py | 6 +- .../availability_strategy.py | 4 +- .../source_salesforce/rate_limiting.py | 4 +- .../source_salesforce/source.py | 8 +- .../source_salesforce/streams.py | 6 +- .../source-salesforce/unit_tests/api_test.py | 80 ++- .../source-salesforce/unit_tests/conftest.py | 8 +- .../integration/test_bulk_stream.py | 105 ++- .../integration/test_rest_stream.py | 64 +- .../unit_tests/integration/test_source.py | 18 +- .../unit_tests/integration/utils.py | 21 +- .../salesforce_job_response_builder.py | 28 +- .../unit_tests/test_availability_strategy.py | 4 +- .../unit_tests/test_rate_limiting.py | 44 +- .../unit_tests/test_slice_generation.py | 8 +- .../connectors/source-salesloft/components.py | 1 - .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-sentry/main.py | 1 + .../source-sentry/source_sentry/run.py | 3 +- .../unit_tests/integration/config_builder.py | 2 +- .../integration/test_events_stream.py | 21 +- .../integration/test_issues_stream.py | 15 +- .../source-sentry/unit_tests/test_streams.py | 24 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/conftest.py | 2 + .../integration_tests/integration_test.py | 18 +- .../integration_tests/utils.py | 1 + .../connectors/source-sftp-bulk/main.py | 1 + .../source_sftp_bulk/client.py | 4 +- .../source_sftp_bulk/source.py | 1 + .../source-sftp-bulk/source_sftp_bulk/spec.py | 3 +- .../source_sftp_bulk/stream_reader.py | 3 +- .../unit_tests/stream_reader_test.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-shopify/main.py | 1 + .../source-shopify/source_shopify/auth.py | 1 - .../source_shopify/config_migrations.py | 3 +- .../source_shopify/http_request.py | 4 +- .../source-shopify/source_shopify/scopes.py | 5 +- .../shopify_graphql/bulk/job.py | 3 +- .../shopify_graphql/bulk/query.py | 1 - .../shopify_graphql/bulk/retry.py | 1 + .../shopify_graphql/bulk/tools.py | 1 + .../source_shopify/shopify_graphql/graphql.py | 1 + .../source_shopify/shopify_graphql/schema.py | 1 + .../source-shopify/source_shopify/source.py | 3 +- .../source_shopify/streams/base_streams.py | 11 +- .../source_shopify/streams/streams.py | 6 +- .../source-shopify/source_shopify/utils.py | 4 +- .../source-shopify/unit_tests/conftest.py | 21 +- .../unit_tests/graphql_bulk/test_job.py | 78 ++- .../unit_tests/graphql_bulk/test_query.py | 209 ++++-- .../unit_tests/graphql_bulk/test_record.py | 2 +- .../integration/api/authentication.py | 6 +- .../unit_tests/integration/api/bulk.py | 74 +- .../integration/test_bulk_stream.py | 116 ++- .../source-shopify/unit_tests/test_auth.py | 1 + .../unit_tests/test_cached_stream_state.py | 1 + .../unit_tests/test_control_rate_limit.py | 1 + .../unit_tests/test_deleted_events_stream.py | 4 +- .../unit_tests/test_graphql_products.py | 23 +- .../test_migrations/test_config_migrations.py | 6 +- .../source-shopify/unit_tests/test_source.py | 49 +- .../source-shopify/unit_tests/unit_test.py | 5 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-slack/main.py | 1 + .../components/channel_members_extractor.py | 1 + .../source_slack/components/join_channels.py | 2 + .../components/slack_backoff_strategy.py | 3 +- .../source_slack/config_migrations.py | 1 + .../source-slack/source_slack/source.py | 1 + .../source-slack/source_slack/streams.py | 3 +- .../source-slack/unit_tests/conftest.py | 40 +- .../unit_tests/test_components.py | 55 +- .../unit_tests/test_config_migrations.py | 1 + .../source-slack/unit_tests/test_source.py | 1 + .../source-slack/unit_tests/test_streams.py | 63 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-smartsheets/main.py | 1 + .../source_smartsheets/sheet.py | 1 + .../source-smartsheets/unit_tests/conftest.py | 1 + .../unit_tests/test_streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-stripe/main.py | 1 + ...emental_stripe_sub_stream_error_handler.py | 1 + .../error_handlers/stripe_error_handler.py | 2 + ...emental_stripe_sub_stream_error_mapping.py | 1 + .../source-stripe/source_stripe/run.py | 3 +- .../source-stripe/source_stripe/source.py | 8 +- .../source-stripe/source_stripe/streams.py | 2 + .../source-stripe/unit_tests/conftest.py | 2 + .../unit_tests/integration/test_accounts.py | 4 +- .../integration/test_application_fees.py | 4 +- .../test_application_fees_refunds.py | 4 +- .../integration/test_authorizations.py | 4 +- .../integration/test_bank_accounts.py | 4 +- .../unit_tests/integration/test_cards.py | 4 +- .../integration/test_early_fraud_warnings.py | 4 +- .../unit_tests/integration/test_events.py | 4 +- .../test_external_account_bank_accounts.py | 4 +- .../test_external_account_cards.py | 4 +- .../integration/test_payment_methods.py | 67 +- .../test_payout_balance_transactions.py | 20 +- .../unit_tests/integration/test_persons.py | 4 +- .../unit_tests/integration/test_reviews.py | 4 +- .../integration/test_transactions.py | 4 +- .../source-stripe/unit_tests/test_source.py | 4 +- .../source-stripe/unit_tests/test_streams.py | 28 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-surveycto/main.py | 1 + .../source_surveycto/source.py | 2 +- .../unit_tests/test_source.py | 8 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-surveymonkey/main.py | 1 + .../source_surveymonkey/config_migrations.py | 1 + .../source_surveymonkey/source.py | 2 + .../source_surveymonkey/streams.py | 2 + .../unit_tests/conftest.py | 24 +- .../unit_tests/test_config_migrations.py | 1 + .../unit_tests/test_custom_router.py | 4 +- .../unit_tests/test_for_updated_state.py | 3 +- .../unit_tests/test_source.py | 1 + .../unit_tests/test_streams.py | 39 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-the-guardian-api/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../unit_tests/test_paginator.py | 35 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-tiktok-marketing/main.py | 1 + .../advertiser_ids_partition_router.py | 1 + .../semi_incremental_record_filter.py | 1 - .../components/transformations.py | 1 - .../source_tiktok_marketing/source.py | 1 + .../integration/advetiser_slices.py | 1 + .../unit_tests/integration/config_builder.py | 9 +- .../integration/test_creative_assets_music.py | 5 +- .../test_creative_assets_portfolios.py | 5 +- .../integration/test_reports_hourly.py | 100 +-- .../unit_tests/test_components.py | 179 ++--- .../unit_tests/test_source.py | 62 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-tplcentral/main.py | 1 + .../source_tplcentral/source.py | 3 +- .../source_tplcentral/streams.py | 1 + .../unit_tests/test_incremental_streams.py | 3 +- .../connectors/source-trello/components.py | 1 - .../integration_tests/acceptance.py | 1 + .../test_order_ids_partition_router.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-twilio/main.py | 1 + .../source-twilio/source_twilio/source.py | 2 + .../source-twilio/source_twilio/streams.py | 6 +- .../source-twilio/unit_tests/test_streams.py | 32 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-typeform/main.py | 1 + .../source-typeform/source_typeform/run.py | 3 +- .../source-typeform/source_typeform/source.py | 1 + .../unit_tests/test_authenticator.py | 11 +- .../test_form_id_partition_router.py | 4 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-us-census/components.py | 2 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-webflow/main.py | 1 + .../source-webflow/source_webflow/source.py | 4 +- .../webflow_to_airbyte_mapping.py | 1 - .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-xero/main.py | 1 + .../source-xero/source_xero/components.py | 1 + .../connectors/source-xero/source_xero/run.py | 3 +- .../source-xero/source_xero/source.py | 1 + .../source-xero/unit_tests/conftest.py | 3 +- .../unit_tests/test_custom_parsing.py | 11 +- .../source-xero/unit_tests/test_source.py | 1 + .../source-xero/unit_tests/test_streams.py | 17 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-yandex-metrica/main.py | 1 + .../source_yandex_metrica/source.py | 2 + .../source_yandex_metrica/streams.py | 4 +- .../unit_tests/test_source.py | 1 + .../unit_tests/test_streams.py | 4 +- .../integration_tests/acceptance.py | 1 + .../connectors/source-younium/components.py | 4 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../source-youtube-analytics/main.py | 1 + .../source_youtube_analytics/source.py | 1 + .../unit_tests/test_source.py | 3 +- .../integration_tests/acceptance.py | 1 + .../build_customization.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-zendesk-chat/main.py | 1 + .../components/bans_record_extractor.py | 1 + .../components/id_offset_pagination.py | 1 + .../components/time_offset_pagination.py | 1 + .../source_zendesk_chat/source.py | 1 + .../unit_tests/components/conftest.py | 29 +- .../components/test_bans_record_extractor.py | 4 +- .../components/test_id_incremental_cursor.py | 36 +- .../components/test_id_offset_pagination.py | 13 +- .../components/test_time_offset_pagination.py | 12 +- .../components/test_timestamp_based_cursor.py | 29 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-zendesk-support/main.py | 1 + .../source_zendesk_support/components.py | 5 +- .../source_zendesk_support/run.py | 3 +- .../source_zendesk_support/source.py | 2 + .../source_zendesk_support/streams.py | 5 +- .../unit_tests/conftest.py | 1 + .../unit_tests/integrations/helpers.py | 31 +- .../unit_tests/integrations/test_groups.py | 6 +- .../integrations/test_post_comment_votes.py | 21 +- .../integrations/test_post_comments.py | 21 +- .../integrations/test_post_votes.py | 21 +- .../integrations/test_ticket_metrics.py | 29 +- .../unit_tests/integrations/utils.py | 8 +- .../zs_requests/base_request_builder.py | 7 +- .../post_comment_votes_request_builder.py | 4 +- .../unit_tests/test_backoff_on_rate_limit.py | 1 + .../unit_tests/test_components.py | 3 +- .../unit_tests/unit_test.py | 148 ++-- .../source-zendesk-talk/components.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../integration_tests/acceptance.py | 1 + .../connectors/source-zenloop/main.py | 1 + .../source_zenloop/components.py | 1 - .../source-zenloop/source_zenloop/source.py | 1 + .../source-zenloop/source_zenloop/streams.py | 3 +- .../integration_tests/acceptance.py | 1 + .../integration_tests/test_stream_factory.py | 1 + .../connectors/source-zoho-crm/main.py | 1 + .../connectors/source-zoho-crm/setup.py | 1 + .../source-zoho-crm/source_zoho_crm/api.py | 1 + .../source-zoho-crm/source_zoho_crm/auth.py | 1 + .../source-zoho-crm/source_zoho_crm/run.py | 3 +- .../source-zoho-crm/source_zoho_crm/source.py | 1 + .../source_zoho_crm/streams.py | 2 + .../source-zoho-crm/unit_tests/parametrize.py | 1 + .../source-zoho-crm/unit_tests/test_auth.py | 1 + .../unit_tests/test_incremental_streams.py | 3 +- .../connectors/source-zoom/components.py | 4 +- .../integration_tests/acceptance.py | 1 + tools/bin/cleanup-workflow-runs.py | 1 + tools/bin/identify-dormant-workflows.py | 2 +- tools/bin/prep_test_results_for_gcs.py | 1 + tools/bin/record_obfuscator.py | 1 + tools/bin/update_intellij_venv.py | 1 + .../schema_generator/infer_schemas.py | 3 +- .../schema_generator/schema_generator/main.py | 1 - 1555 files changed, 7445 insertions(+), 5799 deletions(-) diff --git a/airbyte-ci/connectors/base_images/base_images/commands.py b/airbyte-ci/connectors/base_images/base_images/commands.py index 169420353c701..626724cf61fba 100644 --- a/airbyte-ci/connectors/base_images/base_images/commands.py +++ b/airbyte-ci/connectors/base_images/base_images/commands.py @@ -10,9 +10,10 @@ import dagger import inquirer # type: ignore import semver -from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry from jinja2 import Environment, FileSystemLoader +from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry + async def _generate_docs(dagger_client: dagger.Client): """This function will generate the README.md file from the templates/README.md.j2 template. diff --git a/airbyte-ci/connectors/base_images/base_images/consts.py b/airbyte-ci/connectors/base_images/base_images/consts.py index ce3701c300f1b..cdee55c11a7ee 100644 --- a/airbyte-ci/connectors/base_images/base_images/consts.py +++ b/airbyte-ci/connectors/base_images/base_images/consts.py @@ -2,10 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -"""This module declares constants used by the base_images module. -""" +"""This module declares constants used by the base_images module.""" import dagger + REMOTE_REGISTRY = "docker.io" PLATFORMS_WE_PUBLISH_FOR = (dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64")) diff --git a/airbyte-ci/connectors/base_images/base_images/errors.py b/airbyte-ci/connectors/base_images/base_images/errors.py index 3e009806c80ba..78596195cdb51 100644 --- a/airbyte-ci/connectors/base_images/base_images/errors.py +++ b/airbyte-ci/connectors/base_images/base_images/errors.py @@ -2,8 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -"""This module contains the exceptions used by the base_images module. -""" +"""This module contains the exceptions used by the base_images module.""" from typing import Union diff --git a/airbyte-ci/connectors/base_images/base_images/hacks.py b/airbyte-ci/connectors/base_images/base_images/hacks.py index 1d8ab1a9ad48a..5f3bf76d2a1f3 100644 --- a/airbyte-ci/connectors/base_images/base_images/hacks.py +++ b/airbyte-ci/connectors/base_images/base_images/hacks.py @@ -4,6 +4,7 @@ import dagger + # If we perform addition dagger operations on the container, we need to make sure that a mapping exists for the new field name. DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION = { "from": lambda field: f'FROM {field.args.get("address")}', diff --git a/airbyte-ci/connectors/base_images/base_images/java/bases.py b/airbyte-ci/connectors/base_images/base_images/java/bases.py index ed820e5b98633..72a376e3aae80 100644 --- a/airbyte-ci/connectors/base_images/base_images/java/bases.py +++ b/airbyte-ci/connectors/base_images/base_images/java/bases.py @@ -6,6 +6,7 @@ from typing import Callable, Final import dagger + from base_images import bases, published_image from base_images import sanity_checks as base_sanity_checks from base_images.python import sanity_checks as python_sanity_checks @@ -22,9 +23,9 @@ class AirbyteJavaConnectorBaseImage(bases.AirbyteConnectorBaseImage): DD_AGENT_JAR_URL: Final[str] = "https://dtdg.co/latest-java-tracer" BASE_SCRIPT_URL = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh" - JAVA_BASE_SCRIPT_URL: Final[ - str - ] = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh" + JAVA_BASE_SCRIPT_URL: Final[str] = ( + "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh" + ) def get_container(self, platform: dagger.Platform) -> dagger.Container: """Returns the container used to build the base image for java connectors diff --git a/airbyte-ci/connectors/base_images/base_images/publish.py b/airbyte-ci/connectors/base_images/base_images/publish.py index 4f4ebcfaf3e9c..e6e69fa8b7fec 100644 --- a/airbyte-ci/connectors/base_images/base_images/publish.py +++ b/airbyte-ci/connectors/base_images/base_images/publish.py @@ -4,6 +4,7 @@ import dagger + from base_images import bases, consts, published_image diff --git a/airbyte-ci/connectors/base_images/base_images/python/bases.py b/airbyte-ci/connectors/base_images/base_images/python/bases.py index cedb8eaad3dd6..191c05c20574b 100644 --- a/airbyte-ci/connectors/base_images/base_images/python/bases.py +++ b/airbyte-ci/connectors/base_images/base_images/python/bases.py @@ -6,6 +6,7 @@ from typing import Callable, Final import dagger + from base_images import bases, published_image from base_images import sanity_checks as base_sanity_checks from base_images.python import sanity_checks as python_sanity_checks diff --git a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py index 58724281a1e28..798839009669b 100644 --- a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py @@ -3,6 +3,7 @@ # import dagger + from base_images import errors from base_images import sanity_checks as base_sanity_checks diff --git a/airbyte-ci/connectors/base_images/base_images/root_images.py b/airbyte-ci/connectors/base_images/base_images/root_images.py index dcd0892a8f6cc..0134cd4d34e88 100644 --- a/airbyte-ci/connectors/base_images/base_images/root_images.py +++ b/airbyte-ci/connectors/base_images/base_images/root_images.py @@ -4,6 +4,7 @@ from .published_image import PublishedImage + PYTHON_3_9_18 = PublishedImage( registry="docker.io", repository="python", diff --git a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py index 287636cef73cb..3ea3b4e9b3107 100644 --- a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py +++ b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py @@ -6,6 +6,7 @@ from typing import Optional import dagger + from base_images import errors diff --git a/airbyte-ci/connectors/base_images/base_images/utils/docker.py b/airbyte-ci/connectors/base_images/base_images/utils/docker.py index b8bc51449170c..b7618fcb4aedc 100644 --- a/airbyte-ci/connectors/base_images/base_images/utils/docker.py +++ b/airbyte-ci/connectors/base_images/base_images/utils/docker.py @@ -9,6 +9,7 @@ from typing import List, Tuple import dagger + from base_images import console, published_image @@ -31,7 +32,6 @@ def get_credentials() -> Tuple[str, str]: class CraneClient: - CRANE_IMAGE_ADDRESS = "gcr.io/go-containerregistry/crane/debug:c195f151efe3369874c72662cd69ad43ee485128@sha256:94f61956845714bea3b788445454ae4827f49a90dcd9dac28255c4cccb6220ad" def __init__(self, dagger_client: dagger.Client, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0): diff --git a/airbyte-ci/connectors/base_images/base_images/version_registry.py b/airbyte-ci/connectors/base_images/base_images/version_registry.py index 1495c77c327c4..df2a137a6c179 100644 --- a/airbyte-ci/connectors/base_images/base_images/version_registry.py +++ b/airbyte-ci/connectors/base_images/base_images/version_registry.py @@ -11,6 +11,7 @@ import dagger import semver + from base_images import consts, published_image from base_images.bases import AirbyteConnectorBaseImage from base_images.java.bases import AirbyteJavaConnectorBaseImage @@ -18,6 +19,7 @@ from base_images.utils import docker from connector_ops.utils import ConnectorLanguage # type: ignore + MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage, AirbyteJavaConnectorBaseImage] diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py index 9f2f474568487..6f31626adf7bc 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py @@ -11,6 +11,7 @@ from . import SecretsManager + logger = Logger() ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS" diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py index 067e89da1d68d..84295b70de45d 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py @@ -8,6 +8,7 @@ from dataclasses import dataclass + DEFAULT_SECRET_FILE = "config" diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py index c024caf015870..554d0d3f604f5 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py @@ -17,6 +17,7 @@ from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret + DEFAULT_SECRET_FILE_WITH_EXT = DEFAULT_SECRET_FILE + ".json" GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) diff --git a/airbyte-ci/connectors/common_utils/common_utils/google_api.py b/airbyte-ci/connectors/common_utils/common_utils/google_api.py index 68ff38ae5a9f6..69d12894ac904 100644 --- a/airbyte-ci/connectors/common_utils/common_utils/google_api.py +++ b/airbyte-ci/connectors/common_utils/common_utils/google_api.py @@ -11,6 +11,7 @@ from .logger import Logger + TOKEN_TTL = 3600 diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py index f109c4299c866..b28bbbf0d1a80 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py @@ -5,8 +5,10 @@ from typing import Dict, List, Optional, Set, Tuple, Union import yaml + from connector_ops import utils + # The breaking change reviewers is still in active use. BREAKING_CHANGE_REVIEWERS = {"breaking-change-reviewers"} CERTIFIED_MANIFEST_ONLY_CONNECTOR_REVIEWERS = {"dev-python"} diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 7eb850c4b880d..32e03ebdbcf0c 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -22,6 +22,7 @@ from rich.console import Console from simpleeval import simple_eval + console = Console() DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py index ed400f6ef598c..1587db395bd9b 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py @@ -12,6 +12,7 @@ import dagger from anyio import Semaphore from connector_ops.utils import Connector # type: ignore + from connectors_insights.insights import generate_insights_for_connector from connectors_insights.result_backends import GCSBucket, LocalDir from connectors_insights.utils import gcs_uri_to_bucket_key, get_all_connectors_in_directory, remove_strict_encrypt_suffix diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py index 8cdf36d6b1548..5267c88df8cea 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING import requests + from connectors_insights.hacks import get_ci_on_master_report from connectors_insights.models import ConnectorInsights from connectors_insights.pylint import get_pylint_output diff --git a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py index d8fc5a4105abc..eef3fe1cd0860 100644 --- a/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py +++ b/airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import ConnectorLanguage # type: ignore + from connectors_insights.utils import never_fail_exec if TYPE_CHECKING: diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py index fa0f86795e479..c78308c980d9c 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import Connector # type: ignore + from connectors_qa.models import Check, CheckCategory, CheckResult if TYPE_CHECKING: @@ -27,7 +28,6 @@ class CheckConnectorIconIsAvailable(AssetsCheck): requires_metadata = False def _check_is_valid_svg(self, icon_path: Path) -> Tuple[bool, str | None]: - try: # Ensure the file has an .svg extension if not icon_path.suffix.lower() == ".svg": diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py index 700b7162ed27e..5665882b52ceb 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py @@ -7,9 +7,10 @@ import requests # type: ignore from connector_ops.utils import Connector, ConnectorLanguage # type: ignore -from connectors_qa.models import Check, CheckCategory, CheckResult from pydash.objects import get # type: ignore +from connectors_qa.models import Check, CheckCategory, CheckResult + from .helpers import ( generate_description, prepare_changelog_to_compare, diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py index da2b73b25c728..d7f0cd0c5df72 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py @@ -5,9 +5,10 @@ import toml from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore class MetadataCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py index 806fcb232c2dc..608e9b9dc9fb1 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py @@ -3,9 +3,10 @@ import semver import toml from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from pydash.objects import get # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from pydash.objects import get # type: ignore class PackagingCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py index 61744ef0009a3..6233680225306 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py @@ -4,9 +4,10 @@ from typing import Iterable, Optional, Set, Tuple from connector_ops.utils import Connector, ConnectorLanguage # type: ignore +from pydash.objects import get # type: ignore + from connectors_qa import consts from connectors_qa.models import Check, CheckCategory, CheckResult -from pydash.objects import get # type: ignore class SecurityCheck(Check): diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py index fb70e431fd915..39860d002a74b 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py @@ -2,9 +2,10 @@ from connector_ops.utils import Connector # type: ignore -from connectors_qa.models import Check, CheckCategory, CheckResult from pydash.collections import find # type: ignore +from connectors_qa.models import Check, CheckCategory, CheckResult + class TestingCheck(Check): category = CheckCategory.TESTING diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py index 77ec4a0879d0a..1f680c3c12ea0 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py @@ -6,11 +6,12 @@ import asyncclick as click import asyncer from connector_ops.utils import Connector # type: ignore +from jinja2 import Environment, PackageLoader, select_autoescape + from connectors_qa.checks import ENABLED_CHECKS from connectors_qa.consts import CONNECTORS_QA_DOC_TEMPLATE_NAME from connectors_qa.models import Check, CheckCategory, CheckResult, CheckStatus, Report from connectors_qa.utils import get_all_connectors_in_directory, remove_strict_encrypt_suffix -from jinja2 import Environment, PackageLoader, select_autoescape # HELPERS diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py index 5d6eb29fe615b..5b823a4830d42 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py @@ -11,6 +11,7 @@ from typing import Dict, List, Optional from connector_ops.utils import Connector, ConnectorLanguage # type: ignore + from connectors_qa import consts ALL_LANGUAGES = [ @@ -289,9 +290,9 @@ def to_json(self) -> str: " ", "_" ) connectors_report[connector_technical_name]["badge_text"] = badge_text - connectors_report[connector_technical_name][ - "badge_url" - ] = f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}" + connectors_report[connector_technical_name]["badge_url"] = ( + f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}" + ) return json.dumps( { "generation_timestamp": datetime.utcnow().isoformat(), diff --git a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py index a2ee14c8436ac..1833e1e05ac03 100644 --- a/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py +++ b/airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py @@ -4,6 +4,7 @@ from typing import Set from connector_ops.utils import Connector # type: ignore + from connectors_qa import consts diff --git a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py index afe74946cb106..f28ad4fd8c5af 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py @@ -6,23 +6,28 @@ import git import pytest from asyncclick.testing import CliRunner + from connectors_qa.cli import generate_documentation DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO = Path("docs/contributing-to-airbyte/resources/qa-checks.md") + @pytest.fixture def airbyte_repo(): return git.Repo(search_parent_directories=True) + @pytest.mark.asyncio async def test_generated_qa_checks_documentation_is_up_to_date(airbyte_repo, tmp_path): # Arrange current_doc = (airbyte_repo.working_dir / DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO).read_text() newly_generated_doc_path = tmp_path / "qa-checks.md" - + # Act await CliRunner().invoke(generate_documentation, [str(tmp_path / "qa-checks.md")], catch_exceptions=False) # Assert suggested_command = f"connectors-qa generate-documentation {DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO}" - assert newly_generated_doc_path.read_text() == current_doc, f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." + assert ( + newly_generated_doc_path.read_text() == current_doc + ), f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py index ed4e8eebbfe96..f26d49671fdd5 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py @@ -42,7 +42,6 @@ def test_fail_when_icon_path_is_not_named_icon_svg(self, tmp_path, mocker): assert result.status == CheckStatus.FAILED assert result.message == "Icon file is not a SVG file" - def test_fail_when_icon_file_is_not_valid_svg(self, tmp_path, mocker): # Arrange connector = mocker.MagicMock() diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py index 959ba6b48311e..f5f58912489a7 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py @@ -196,7 +196,6 @@ def test_pass_when_documentation_file_path_exists(self, mocker, tmp_path): class TestCheckDocumentationContent: - def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_path): # Arrange connector = mocker.Mock( @@ -204,7 +203,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat ab_internal_sl=300, language="python", connector_type="source", - documentation_file_path=tmp_path / "not_existing_documentation.md" + documentation_file_path=tmp_path / "not_existing_documentation.md", ) # Act @@ -217,11 +216,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat def test_fail_when_documentation_file_path_is_none(self, mocker): # Arrange connector = mocker.Mock( - technical_name="test-connector", - ab_internal_sl=300, - language="python", - connector_type="source", - documentation_file_path=None + technical_name="test-connector", ab_internal_sl=300, language="python", connector_type="source", documentation_file_path=None ) # Act @@ -263,34 +258,28 @@ def test_fail_when_documentation_file_has_missing_headers(self, connector_with_i assert "Actual Heading: 'For Airbyte Cloud:'. Expected Heading: 'Setup guide'" in result.message def test_fail_when_documentation_file_not_have_all_required_fields_in_prerequisites_section_content( - self, - connector_with_invalid_documentation + self, connector_with_invalid_documentation ): # Act - result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run( - connector_with_invalid_documentation - ) + result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Missing descriptions for required spec fields: github repositories" in result.message - def test_fail_when_documentation_file_has_invalid_source_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_source_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector GitHub section content does not follow standard template:" in result.message - assert "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." in result.message + assert ( + "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." + in result.message + ) - def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_invalid_documentation) @@ -299,10 +288,7 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_cont assert "Connector For Airbyte Cloud: section content does not follow standard template:" in result.message assert "+ 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account." in result.message - def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_invalid_documentation) @@ -311,23 +297,19 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_conte assert "Connector For Airbyte Open Source: section content does not follow standard template" in result.message assert "+ 1. Navigate to the Airbyte Open Source dashboard." in result.message - def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector Supported sync modes section content does not follow standard template:" in result.message - assert ("+ The GitHub source connector supports the following" - " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):") in result.message + assert ( + "+ The GitHub source connector supports the following" + " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):" + ) in result.message - def test_fail_when_documentation_file_has_invalid_tutorials_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_tutorials_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_invalid_documentation) @@ -336,10 +318,7 @@ def test_fail_when_documentation_file_has_invalid_tutorials_section_content( assert "Connector Tutorials section content does not follow standard template:" in result.message assert "+ Now that you have set up the GitHub source connector, check out the following GitHub tutorials:" in result.message - def test_fail_when_documentation_file_has_invalid_changelog_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_changelog_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_invalid_documentation) @@ -356,10 +335,7 @@ def test_pass_when_documentation_file_has_correct_headers(self, connector_with_c assert result.status == CheckStatus.PASSED assert result.message == "Documentation guidelines are followed" - def test_pass_when_documentation_file_has_correct_prerequisites_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_prerequisites_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_correct_documentation) @@ -367,10 +343,7 @@ def test_pass_when_documentation_file_has_correct_prerequisites_section_content( assert result.status == CheckStatus.PASSED assert "All required fields from spec are present in the connector documentation" in result.message - def test_pass_when_documentation_file_has_correct_source_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_source_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_correct_documentation) @@ -378,10 +351,7 @@ def test_pass_when_documentation_file_has_correct_source_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_correct_documentation) @@ -389,10 +359,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_cont assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_correct_documentation) @@ -400,10 +367,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_conte assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_correct_documentation) @@ -411,10 +375,7 @@ def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_c assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_tutorials_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_tutorials_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_correct_documentation) @@ -422,10 +383,7 @@ def test_pass_when_documentation_file_has_correct_tutorials_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_headers_order( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_headers_order(self, connector_with_correct_documentation): # Act result = documentation.CheckDocumentationHeadersOrder()._run(connector_with_correct_documentation) @@ -433,10 +391,7 @@ def test_pass_when_documentation_file_has_correct_headers_order( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_changelog_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_changelog_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_correct_documentation) diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py index b3b6e952fa44d..482a2536b2414 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py @@ -4,6 +4,7 @@ import os import pytest + from connectors_qa import consts from connectors_qa.checks import metadata from connectors_qa.models import CheckStatus diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py index b494c9d400741..56f98ae27081c 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py @@ -133,6 +133,7 @@ def test_pass_if_publish_to_pypi_is_disabled(self, mocker): assert result.status == CheckStatus.PASSED assert "PyPi publishing is declared" in result.message + class TestCheckConnectorLicense: def test_fail_when_license_is_missing(self, mocker): # Arrange diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py index c6c997b03fd69..99a90b2cb2304 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py @@ -2,6 +2,7 @@ import pytest from connector_ops.utils import ConnectorLanguage + from connectors_qa.checks import testing from connectors_qa.models import CheckStatus @@ -56,9 +57,7 @@ "connectorTestSuitesOptions": [ { "suite": testing.AcceptanceTestsEnabledCheck.test_suite_name, - "testSecrets": { - "testSecret": "test" - }, + "testSecrets": {"testSecret": "test"}, }, { "suite": "unit", @@ -71,10 +70,10 @@ OTHER_USAGE_VALUES = ["low", "none", "unknown", None, ""] DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES = [ - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, ] DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES = [ @@ -89,21 +88,9 @@ class TestAcceptanceTestsEnabledCheck: @pytest.mark.parametrize( "cases_to_test, usage_values_to_test, expected_result", [ - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - OTHER_USAGE_VALUES, - CheckStatus.SKIPPED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.PASSED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.FAILED - ) + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, OTHER_USAGE_VALUES, CheckStatus.SKIPPED), + (DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.PASSED), + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.FAILED), ], ) def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases_to_test, usage_values_to_test, expected_result): @@ -115,11 +102,13 @@ def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases metadata=metadata_case, language=ConnectorLanguage.PYTHON, connector_type="source", - ab_internal_sl=100 + ab_internal_sl=100, ) # Act result = testing.AcceptanceTestsEnabledCheck().run(connector) # Assert - assert result.status == expected_result, f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" + assert ( + result.status == expected_result + ), f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py index 442a038f95950..bcb3198e7bca2 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from connector_ops.utils import ConnectorLanguage + from connectors_qa import consts from connectors_qa.checks import ENABLED_CHECKS from connectors_qa.models import CheckStatus diff --git a/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py b/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py index 1e80a737c6a22..116e20e1dfe0b 100644 --- a/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py +++ b/airbyte-ci/connectors/erd/src/erd/dbml_assembler.py @@ -4,11 +4,22 @@ from typing import List, Set, Union import yaml -from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver -from airbyte_protocol.models import AirbyteCatalog, AirbyteStream # type: ignore # missing library stubs or py.typed marker -from erd.relationships import Relationships +from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ( + ManifestReferenceResolver, +) +from airbyte_protocol.models import ( # type: ignore # missing library stubs or py.typed marker + AirbyteCatalog, + AirbyteStream, +) from pydbml import Database # type: ignore # missing library stubs or py.typed marker -from pydbml.classes import Column, Index, Reference, Table # type: ignore # missing library stubs or py.typed marker +from pydbml.classes import ( # type: ignore # missing library stubs or py.typed marker + Column, + Index, + Reference, + Table, +) + +from erd.relationships import Relationships class Source: @@ -25,33 +36,67 @@ def is_dynamic(self, stream_name: str) -> bool: manifest_static_streams = set() if self._has_manifest(): with open(self._get_manifest_path()) as manifest_file: - resolved_manifest = ManifestReferenceResolver().preprocess_manifest(yaml.safe_load(manifest_file)) + resolved_manifest = ManifestReferenceResolver().preprocess_manifest( + yaml.safe_load(manifest_file) + ) for stream in resolved_manifest["streams"]: if "schema_loader" not in stream: # stream is assumed to have `DefaultSchemaLoader` which will show in the schemas folder so we can skip continue if stream["schema_loader"]["type"] == "InlineSchemaLoader": - name = stream["name"] if "name" in stream else stream.get("$parameters").get("name", None) + name = ( + stream["name"] + if "name" in stream + else stream.get("$parameters").get("name", None) + ) if not name: print(f"Could not retrieve name for this stream: {stream}") continue - manifest_static_streams.add(stream["name"] if "name" in stream else stream.get("$parameters").get("name", None)) + manifest_static_streams.add( + stream["name"] + if "name" in stream + else stream.get("$parameters").get("name", None) + ) - return stream_name not in manifest_static_streams | self._get_streams_from_schemas_folder() + return ( + stream_name + not in manifest_static_streams | self._get_streams_from_schemas_folder() + ) def _get_streams_from_schemas_folder(self) -> Set[str]: - schemas_folder = self._source_folder / self._source_technical_name.replace("-", "_") / "schemas" - return {p.name.replace(".json", "") for p in schemas_folder.iterdir() if p.is_file()} if schemas_folder.exists() else set() + schemas_folder = ( + self._source_folder + / self._source_technical_name.replace("-", "_") + / "schemas" + ) + return ( + { + p.name.replace(".json", "") + for p in schemas_folder.iterdir() + if p.is_file() + } + if schemas_folder.exists() + else set() + ) def _get_manifest_path(self) -> Path: - return self._source_folder / self._source_technical_name.replace("-", "_") / "manifest.yaml" + return ( + self._source_folder + / self._source_technical_name.replace("-", "_") + / "manifest.yaml" + ) def _has_manifest(self) -> bool: return self._get_manifest_path().exists() class DbmlAssembler: - def assemble(self, source: Source, discovered_catalog: AirbyteCatalog, relationships: Relationships) -> Database: + def assemble( + self, + source: Source, + discovered_catalog: AirbyteCatalog, + relationships: Relationships, + ) -> Database: database = Database() for stream in discovered_catalog.streams: if source.is_dynamic(stream.name): @@ -66,7 +111,9 @@ def assemble(self, source: Source, discovered_catalog: AirbyteCatalog, relations def _create_table(self, stream: AirbyteStream) -> Table: dbml_table = Table(stream.name) - for property_name, property_information in stream.json_schema.get("properties").items(): + for property_name, property_information in stream.json_schema.get( + "properties" + ).items(): try: dbml_table.add_column( Column( @@ -79,12 +126,20 @@ def _create_table(self, stream: AirbyteStream) -> Table: print(f"Ignoring field {property_name}: {exception}") continue - if stream.source_defined_primary_key and len(stream.source_defined_primary_key) > 1: + if ( + stream.source_defined_primary_key + and len(stream.source_defined_primary_key) > 1 + ): if any(map(lambda key: len(key) != 1, stream.source_defined_primary_key)): - raise ValueError(f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`") + raise ValueError( + f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`" + ) composite_key_columns = [ - column for key in stream.source_defined_primary_key for column in dbml_table.columns if column.name in key + column + for key in stream.source_defined_primary_key + for column in dbml_table.columns + if column.name in key ] if len(composite_key_columns) < len(stream.source_defined_primary_key): raise ValueError("Unexpected error: missing PK column from dbml table") @@ -97,11 +152,15 @@ def _create_table(self, stream: AirbyteStream) -> Table: ) return dbml_table - def _add_references(self, source: Source, database: Database, relationships: Relationships) -> None: + def _add_references( + self, source: Source, database: Database, relationships: Relationships + ) -> None: for stream in relationships["streams"]: for column_name, relationship in stream["relations"].items(): if source.is_dynamic(stream["name"]): - print(f"Skipping relationship as stream {stream['name']} from relationship is dynamic") + print( + f"Skipping relationship as stream {stream['name']} from relationship is dynamic" + ) continue try: @@ -109,18 +168,26 @@ def _add_references(self, source: Source, database: Database, relationships: Rel ".", 1 ) # we support the field names having dots but not stream name hence we split on the first dot only except ValueError as exception: - raise ValueError(f"Could not handle relationship {relationship}") from exception + raise ValueError( + f"Could not handle relationship {relationship}" + ) from exception if source.is_dynamic(target_table_name): - print(f"Skipping relationship as target stream {target_table_name} is dynamic") + print( + f"Skipping relationship as target stream {target_table_name} is dynamic" + ) continue try: database.add_reference( Reference( type="<>", # we don't have the information of which relationship type it is so we assume many-to-many for now - col1=self._get_column(database, stream["name"], column_name), - col2=self._get_column(database, target_table_name, target_column_name), + col1=self._get_column( + database, stream["name"], column_name + ), + col2=self._get_column( + database, target_table_name, target_column_name + ), ) ) except ValueError as exception: @@ -136,24 +203,38 @@ def _extract_type(self, property_type: Union[str, List[str]]) -> str: # show this in DBML types.remove("null") if len(types) != 1: - raise ValueError(f"Expected only one type apart from `null` but got {len(types)}: {property_type}") + raise ValueError( + f"Expected only one type apart from `null` but got {len(types)}: {property_type}" + ) return types[0] def _is_pk(self, stream: AirbyteStream, property_name: str) -> bool: return stream.source_defined_primary_key == [[property_name]] - def _get_column(self, database: Database, table_name: str, column_name: str) -> Column: - matching_tables = list(filter(lambda dbml_table: dbml_table.name == table_name, database.tables)) + def _get_column( + self, database: Database, table_name: str, column_name: str + ) -> Column: + matching_tables = list( + filter(lambda dbml_table: dbml_table.name == table_name, database.tables) + ) if len(matching_tables) == 0: raise ValueError(f"Could not find table {table_name}") elif len(matching_tables) > 1: - raise ValueError(f"Unexpected error: many tables found with name {table_name}") + raise ValueError( + f"Unexpected error: many tables found with name {table_name}" + ) table: Table = matching_tables[0] - matching_columns = list(filter(lambda column: column.name == column_name, table.columns)) + matching_columns = list( + filter(lambda column: column.name == column_name, table.columns) + ) if len(matching_columns) == 0: - raise ValueError(f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}") + raise ValueError( + f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}" + ) elif len(matching_columns) > 1: - raise ValueError(f"Unexpected error: many columns found with name {column_name} for table {table_name}") + raise ValueError( + f"Unexpected error: many columns found with name {column_name} for table {table_name}" + ) return matching_columns[0] diff --git a/airbyte-ci/connectors/erd/src/erd/erd_service.py b/airbyte-ci/connectors/erd/src/erd/erd_service.py index aff60bd79a9e1..47591d365aaee 100644 --- a/airbyte-ci/connectors/erd/src/erd/erd_service.py +++ b/airbyte-ci/connectors/erd/src/erd/erd_service.py @@ -7,11 +7,16 @@ import dpath import google.generativeai as genai # type: ignore # missing library stubs or py.typed marker -from airbyte_protocol.models import AirbyteCatalog # type: ignore # missing library stubs or py.typed marker +from airbyte_protocol.models import ( + AirbyteCatalog, # type: ignore # missing library stubs or py.typed marker +) +from markdown_it import MarkdownIt +from pydbml.renderer.dbml.default import ( + DefaultDBMLRenderer, # type: ignore # missing library stubs or py.typed marker +) + from erd.dbml_assembler import DbmlAssembler, Source from erd.relationships import Relationships, RelationshipsMerger -from markdown_it import MarkdownIt -from pydbml.renderer.dbml.default import DefaultDBMLRenderer # type: ignore # missing library stubs or py.typed marker class ErdService: @@ -21,12 +26,18 @@ def __init__(self, source_technical_name: str, source_path: Path) -> None: self._model = genai.GenerativeModel("gemini-1.5-flash") if not self._discovered_catalog_path.exists(): - raise ValueError(f"Could not find discovered catalog at path {self._discovered_catalog_path}") + raise ValueError( + f"Could not find discovered catalog at path {self._discovered_catalog_path}" + ) def generate_estimated_relationships(self) -> None: normalized_catalog = self._normalize_schema_catalog(self._get_catalog()) - estimated_relationships = self._get_relations_from_gemini(source_name=self._source_path.name, catalog=normalized_catalog) - with open(self._estimated_relationships_file, "w") as estimated_relationship_file: + estimated_relationships = self._get_relations_from_gemini( + source_name=self._source_path.name, catalog=normalized_catalog + ) + with open( + self._estimated_relationships_file, "w" + ) as estimated_relationship_file: json.dump(estimated_relationships, estimated_relationship_file, indent=4) def write_dbml_file(self) -> None: @@ -34,7 +45,8 @@ def write_dbml_file(self) -> None: Source(self._source_path, self._source_technical_name), self._get_catalog(), RelationshipsMerger().merge( - self._get_relationships(self._estimated_relationships_file), self._get_relationships(self._confirmed_relationships_file) + self._get_relationships(self._estimated_relationships_file), + self._get_relationships(self._confirmed_relationships_file), ), ) @@ -53,13 +65,19 @@ def _normalize_schema_catalog(catalog: AirbyteCatalog) -> dict[str, Any]: to_rem = dpath.search( stream["json_schema"]["properties"], ["**"], - afilter=lambda x: isinstance(x, dict) and ("array" in str(x.get("type", "")) or "object" in str(x.get("type", ""))), + afilter=lambda x: isinstance(x, dict) + and ( + "array" in str(x.get("type", "")) + or "object" in str(x.get("type", "")) + ), ) for key in to_rem: stream["json_schema"]["properties"].pop(key) return streams # type: ignore # as this comes from an AirbyteCatalog dump, the format should be fine - def _get_relations_from_gemini(self, source_name: str, catalog: dict[str, Any]) -> Relationships: + def _get_relations_from_gemini( + self, source_name: str, catalog: dict[str, Any] + ) -> Relationships: """ :param source_name: @@ -74,9 +92,7 @@ def _get_relations_from_gemini(self, source_name: str, catalog: dict[str, Any]) The current JSON Schema format is as follows: {current_schema}, where "streams" has a list of streams, which represents database tables, and list of properties in each, which in turn, represent DB columns. Streams presented in list are the only available ones. Generate and add a `foreign_key` with reference for each field in top level of properties that is helpful in understanding what the data represents and how are streams related to each other. Pay attention to fields ends with '_id'. - """.format( - source_name=source_name, current_schema=catalog - ) + """.format(source_name=source_name, current_schema=catalog) task = """ Please provide answer in the following format: {streams: [{"name": "", "relations": {"": ""} }]} diff --git a/airbyte-ci/connectors/erd/src/erd/relationships.py b/airbyte-ci/connectors/erd/src/erd/relationships.py index 704af25d489e8..432b19981124c 100644 --- a/airbyte-ci/connectors/erd/src/erd/relationships.py +++ b/airbyte-ci/connectors/erd/src/erd/relationships.py @@ -16,16 +16,28 @@ class Relationship(TypedDict): class RelationshipsMerger: - def merge(self, estimated_relationships: Relationships, confirmed_relationships: Relationships) -> Relationships: + def merge( + self, + estimated_relationships: Relationships, + confirmed_relationships: Relationships, + ) -> Relationships: streams = [] for estimated_stream in estimated_relationships["streams"]: - confirmed_relationships_for_stream = self._get_stream(confirmed_relationships, estimated_stream["name"]) + confirmed_relationships_for_stream = self._get_stream( + confirmed_relationships, estimated_stream["name"] + ) if confirmed_relationships_for_stream: - streams.append(self._merge_for_stream(estimated_stream, confirmed_relationships_for_stream)) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None + streams.append( + self._merge_for_stream( + estimated_stream, confirmed_relationships_for_stream + ) + ) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None else: streams.append(estimated_stream) - already_processed_streams = set(map(lambda relationship: relationship["name"], streams)) + already_processed_streams = set( + map(lambda relationship: relationship["name"], streams) + ) for confirmed_stream in confirmed_relationships["streams"]: if confirmed_stream["name"] not in already_processed_streams: streams.append( @@ -36,13 +48,20 @@ def merge(self, estimated_relationships: Relationships, confirmed_relationships: ) return {"streams": streams} - def _merge_for_stream(self, estimated: Relationship, confirmed: Relationship) -> Relationship: + def _merge_for_stream( + self, estimated: Relationship, confirmed: Relationship + ) -> Relationship: relations = copy.deepcopy(confirmed.get("relations", {})) # get estimated but filter out false positives for field, target in estimated.get("relations", {}).items(): - false_positives = confirmed["false_positives"] if "false_positives" in confirmed else {} - if field not in relations and (field not in false_positives or false_positives.get(field, None) != target): # type: ignore # at this point, false_positives should not be None + false_positives = ( + confirmed["false_positives"] if "false_positives" in confirmed else {} + ) + if field not in relations and ( + field not in false_positives + or false_positives.get(field, None) != target + ): # type: ignore # at this point, false_positives should not be None relations[field] = target return { @@ -50,7 +69,9 @@ def _merge_for_stream(self, estimated: Relationship, confirmed: Relationship) -> "relations": relations, } - def _get_stream(self, relationships: Relationships, stream_name: str) -> Optional[Relationship]: + def _get_stream( + self, relationships: Relationships, stream_name: str + ) -> Optional[Relationship]: for stream in relationships["streams"]: if stream.get("name", None) == stream_name: return stream diff --git a/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py b/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py index c697505378cd5..93c65a7025350 100644 --- a/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py +++ b/airbyte-ci/connectors/erd/tests/test_dbml_assembler.py @@ -4,6 +4,7 @@ from unittest.mock import Mock from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, SyncMode + from erd.dbml_assembler import DbmlAssembler, Source from tests.builder import RelationshipBuilder @@ -18,7 +19,9 @@ def setUp(self) -> None: def test_given_no_streams_then_database_is_empty(self) -> None: dbml = self._assembler.assemble( - self._source, AirbyteCatalog(streams=[]), {"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]} + self._source, + AirbyteCatalog(streams=[]), + {"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]}, ) assert not dbml.tables diff --git a/airbyte-ci/connectors/erd/tests/test_relationships.py b/airbyte-ci/connectors/erd/tests/test_relationships.py index f8675cb6bd57f..968928eb17639 100644 --- a/airbyte-ci/connectors/erd/tests/test_relationships.py +++ b/airbyte-ci/connectors/erd/tests/test_relationships.py @@ -17,32 +17,72 @@ def setUp(self) -> None: self._merger = RelationshipsMerger() def test_given_no_confirmed_then_return_estimation(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } confirmed: Relationships = {"streams": []} merged = self._merger.merge(estimated, confirmed) assert merged == estimated - def test_given_confirmed_as_false_positive_then_remove_from_estimation(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_false_positive(_A_COLUMN, _A_TARGET).build()]} + def test_given_confirmed_as_false_positive_then_remove_from_estimation( + self, + ) -> None: + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_false_positive(_A_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) assert merged == {"streams": [{"name": "a_stream_name", "relations": {}}]} - def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives(self) -> None: + def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives( + self, + ) -> None: estimated: Relationships = {"streams": []} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) assert merged == confirmed def test_given_different_columns_then_return_both(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_ANOTHER_COLUMN, _A_TARGET).build()]} + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_ANOTHER_COLUMN, _A_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) @@ -58,9 +98,23 @@ def test_given_different_columns_then_return_both(self) -> None: ] } - def test_given_same_column_but_different_value_then_prioritize_confirmed(self) -> None: - estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]} - confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _ANOTHER_TARGET).build()]} + def test_given_same_column_but_different_value_then_prioritize_confirmed( + self, + ) -> None: + estimated: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _A_TARGET) + .build() + ] + } + confirmed: Relationships = { + "streams": [ + RelationshipBuilder(_A_STREAM_NAME) + .with_relationship(_A_COLUMN, _ANOTHER_TARGET) + .build() + ] + } merged = self._merger.merge(estimated, confirmed) diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py index 50a0209655cbb..303799e245ebb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py @@ -13,5 +13,4 @@ class BaseBackend(ABC): """ @abstractmethod - def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: - ... + def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: ... diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py index cd6d61ee5d6cd..204f1c099fde5 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py @@ -9,6 +9,7 @@ import duckdb from airbyte_protocol.models import AirbyteMessage # type: ignore + from live_tests.commons.backends.file_backend import FileBackend diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py index a4d0b57c910a5..ae3e68c228f85 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py @@ -10,6 +10,7 @@ from airbyte_protocol.models import AirbyteMessage # type: ignore from airbyte_protocol.models import Type as AirbyteMessageType from cachetools import LRUCache, cached + from live_tests.commons.backends.base_backend import BaseBackend from live_tests.commons.utils import sanitize_stream_name @@ -123,7 +124,11 @@ def _get_filepaths_and_messages(self, message: AirbyteMessage) -> tuple[tuple[st stream_file_path_data_only = self.record_per_stream_directory / f"{sanitize_stream_name(stream_name)}_data_only.jsonl" self.record_per_stream_paths[stream_name] = stream_file_path self.record_per_stream_paths_data_only[stream_name] = stream_file_path_data_only - return (self.RELATIVE_RECORDS_PATH, str(stream_file_path), str(stream_file_path_data_only),), ( + return ( + self.RELATIVE_RECORDS_PATH, + str(stream_file_path), + str(stream_file_path_data_only), + ), ( message.json(sort_keys=True), message.json(sort_keys=True), json.dumps(message.record.data, sort_keys=True), diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py index adcb1c6868cb1..5e2172acf3989 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py @@ -10,6 +10,7 @@ import rich from connection_retriever import ConnectionObject, retrieve_objects # type: ignore from connection_retriever.errors import NotPermittedError # type: ignore + from live_tests.commons.models import ConnectionSubset from live_tests.commons.utils import build_connection_url diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py index 9f32635ce1817..5b2e5efe16755 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py @@ -15,6 +15,7 @@ import anyio import asyncer import dagger + from live_tests.commons import errors from live_tests.commons.models import Command, ExecutionInputs, ExecutionResult from live_tests.commons.proxy import Proxy diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py index f762a2f94ad2e..d58e88c04026a 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import _collections_abc import json import logging import tempfile @@ -13,17 +14,21 @@ from pathlib import Path from typing import Any, Dict, List, Optional -import _collections_abc import dagger import requests -from airbyte_protocol.models import AirbyteCatalog # type: ignore -from airbyte_protocol.models import AirbyteMessage # type: ignore -from airbyte_protocol.models import AirbyteStateMessage # type: ignore -from airbyte_protocol.models import AirbyteStreamStatusTraceMessage # type: ignore -from airbyte_protocol.models import ConfiguredAirbyteCatalog # type: ignore -from airbyte_protocol.models import TraceType # type: ignore +from airbyte_protocol.models import ( + AirbyteCatalog, # type: ignore + AirbyteMessage, # type: ignore + AirbyteStateMessage, # type: ignore + AirbyteStreamStatusTraceMessage, # type: ignore + ConfiguredAirbyteCatalog, # type: ignore + TraceType, # type: ignore +) from airbyte_protocol.models import Type as AirbyteMessageType from genson import SchemaBuilder # type: ignore +from mitmproxy import http +from pydantic import ValidationError + from live_tests.commons.backends import DuckDbBackend, FileBackend from live_tests.commons.secret_access import get_airbyte_api_key from live_tests.commons.utils import ( @@ -33,8 +38,6 @@ sanitize_stream_name, sort_dict_keys, ) -from mitmproxy import http -from pydantic import ValidationError class UserDict(_collections_abc.MutableMapping): # type: ignore diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py index a9e2f6cd54fdb..6b11b405c9cc4 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py @@ -15,6 +15,8 @@ from airbyte_protocol.models import AirbyteCatalog, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification # type: ignore from connection_retriever.audit_logging import get_user_email # type: ignore from connection_retriever.retrieval import ConnectionNotFoundError, NotPermittedError, get_current_docker_image_tag # type: ignore +from rich.prompt import Confirm, Prompt + from live_tests import stash_keys from live_tests.commons.connection_objects_retrieval import ConnectionObject, InvalidConnectionError, get_connection_objects from live_tests.commons.connector_runner import ConnectorRunner, Proxy @@ -35,7 +37,6 @@ from live_tests.commons.utils import build_connection_url, clean_up_artifacts from live_tests.report import Report, ReportState from live_tests.utils import get_catalog, get_spec -from rich.prompt import Confirm, Prompt if TYPE_CHECKING: from _pytest.config import Config diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py index 443ab2f1ac4de..7467fed152b75 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py @@ -5,6 +5,7 @@ import pytest from airbyte_protocol.models import Status, Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py index 61fac41479863..56f955383dbfa 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py @@ -7,6 +7,7 @@ import pytest from _pytest.fixtures import SubRequest from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_catalog diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py index 4530d70086606..3587bb586a237 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py @@ -8,6 +8,7 @@ import pytest from airbyte_protocol.models import AirbyteMessage # type: ignore from deepdiff import DeepDiff # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_test_logger, write_string_to_test_artifact diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py index 967698f7462aa..7a1985fd3e408 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py @@ -5,6 +5,7 @@ import pytest from airbyte_protocol.models import Type # type: ignore + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/report.py b/airbyte-ci/connectors/live-tests/src/live_tests/report.py index 255843f661434..4320d164619ad 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/report.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/report.py @@ -14,6 +14,7 @@ import requests import yaml from jinja2 import Environment, PackageLoader, select_autoescape + from live_tests import stash_keys from live_tests.consts import MAX_LINES_IN_REPORT @@ -21,6 +22,7 @@ import pytest from _pytest.config import Config from airbyte_protocol.models import SyncMode, Type # type: ignore + from live_tests.commons.models import Command, ExecutionResult diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py b/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py index 493182e264d0b..f93488c214bf8 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest + from live_tests.commons.evaluation_modes import TestEvaluationMode from live_tests.commons.models import ConnectionObjects, ConnectionSubset from live_tests.report import Report diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/utils.py b/airbyte-ci/connectors/live-tests/src/live_tests/utils.py index 8c8e039ebbb59..3db571751cc4e 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/utils.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/utils.py @@ -11,11 +11,12 @@ import pytest from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, ConnectorSpecification, Status, Type # type: ignore from deepdiff import DeepDiff # type: ignore +from mitmproxy import http, io # type: ignore +from mitmproxy.addons.savehar import SaveHar # type: ignore + from live_tests import stash_keys from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT -from mitmproxy import http, io # type: ignore -from mitmproxy.addons.savehar import SaveHar # type: ignore if TYPE_CHECKING: from _pytest.fixtures import SubRequest diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py index fc5fd2d7813b7..ac945b830ffbe 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py @@ -7,6 +7,7 @@ import pytest from airbyte_protocol.models import Type + from live_tests.commons.models import ExecutionResult from live_tests.consts import MAX_LINES_IN_REPORT from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py index 44538edaded29..73e4975acbd7c 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py @@ -9,6 +9,7 @@ import jsonschema import pytest from airbyte_protocol.models import AirbyteCatalog + from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py index 16f176eb4e4e1..5d04f883960f8 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py @@ -16,6 +16,7 @@ AirbyteStreamStatusTraceMessage, ConfiguredAirbyteCatalog, ) + from live_tests.commons.json_schema_helper import conforms_to_schema from live_tests.commons.models import ExecutionResult from live_tests.utils import fail_test_on_failing_execution_results, get_test_logger diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py index e1619023b63ff..ae573019bf2a8 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py @@ -9,6 +9,7 @@ import jsonschema import pytest from airbyte_protocol.models import ConnectorSpecification + from live_tests.commons.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_paths_in_connector_config from live_tests.commons.models import ExecutionResult, SecretDict from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema, get_test_logger diff --git a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py index be22da351d93e..fbd8f03bc4f7f 100644 --- a/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py +++ b/airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py @@ -13,6 +13,7 @@ Status, ) from airbyte_protocol.models import Type as AirbyteMessageType + from live_tests.commons.backends import FileBackend diff --git a/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py b/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py index b9fec6c283368..6b73fbfa3ceb6 100644 --- a/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py +++ b/airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py @@ -16,6 +16,8 @@ SyncMode, Type, ) +from pydantic import BaseModel + from live_tests.commons.json_schema_helper import ( ComparableType, JsonSchemaHelper, @@ -23,7 +25,6 @@ get_expected_schema_structure, get_object_structure, ) -from pydantic import BaseModel def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py index 795702333793d..879bc8459ca74 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py @@ -5,6 +5,8 @@ import pathlib import click +from pydantic import ValidationError + from metadata_service.constants import METADATA_FILE_NAME from metadata_service.gcs_upload import ( MetadataDeleteInfo, @@ -14,7 +16,6 @@ upload_metadata_to_gcs, ) from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load -from pydantic import ValidationError def log_metadata_upload_info(metadata_upload_info: MetadataUploadInfo): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py index 4775f25cdd9ab..7cd3d94a3d1cd 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py @@ -16,6 +16,9 @@ import yaml from google.cloud import storage from google.oauth2 import service_account +from pydash import set_ +from pydash.objects import get + from metadata_service.constants import ( COMPONENTS_PY_FILE_NAME, COMPONENTS_ZIP_FILE_NAME, @@ -34,8 +37,6 @@ from metadata_service.models.generated.GitInfo import GitInfo from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load -from pydash import set_ -from pydash.objects import get # 🧩 TYPES diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py index b235780619530..3866a276bf7cf 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py @@ -8,11 +8,12 @@ import semver import yaml -from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from pydantic import ValidationError from pydash.objects import get +from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 + @dataclass(frozen=True) class ValidatorOptions: @@ -247,7 +248,6 @@ def validate_rc_suffix_and_rollout_configuration( if docker_image_tag is None: return False, "The dockerImageTag field is not set." try: - is_major_release_candidate_version = check_is_major_release_candidate_version(docker_image_tag) is_dev_version = check_is_dev_version(docker_image_tag) is_rc_version = check_is_release_candidate_version(docker_image_tag) diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py index 4a821bb318705..e57054eb879ad 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py @@ -6,11 +6,12 @@ import pytest from click.testing import CliRunner +from pydantic import BaseModel, ValidationError, error_wrappers +from test_gcs_upload import stub_is_image_on_docker_hub + from metadata_service import commands from metadata_service.gcs_upload import MetadataUploadInfo, UploadedFile from metadata_service.validators.metadata_validator import ValidatorOptions, validate_docker_image_tag_is_not_decremented -from pydantic import BaseModel, ValidationError, error_wrappers -from test_gcs_upload import stub_is_image_on_docker_hub NOT_TEST_VALIDATORS = [ # Not testing validate_docker_image_tag_is_not_decremented as its tested independently in test_validators @@ -19,6 +20,7 @@ PATCHED_VALIDATORS = [v for v in commands.PRE_UPLOAD_VALIDATORS if v not in NOT_TEST_VALIDATORS] + # TEST VALIDATE COMMAND def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path): runner = CliRunner() diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py index 58a21c52ae2f6..e5e30549d1161 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py @@ -6,6 +6,7 @@ import warnings import pytest + from metadata_service import docker_hub diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py index f3f9f7c0fae6d..6e8cf8abc66d6 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py @@ -7,6 +7,8 @@ import pytest import yaml +from pydash.objects import get + from metadata_service import gcs_upload from metadata_service.constants import ( COMPONENTS_PY_FILE_NAME, @@ -19,7 +21,6 @@ from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import ValidatorOptions -from pydash.objects import get MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["99.99.99", "0.0.0"] MISSING_SHA = "MISSINGSHA" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py index 9ce15092fcb7a..eac4bfd2cde27 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py @@ -5,15 +5,16 @@ from unittest.mock import patch import pytest + from metadata_service.spec_cache import CachedSpec, Registries, SpecCache, get_docker_info_from_spec_cache_path @pytest.fixture def mock_spec_cache(): - with patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, patch( - "google.cloud.storage.Client.bucket" - ) as MockBucket: - + with ( + patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, + patch("google.cloud.storage.Client.bucket") as MockBucket, + ): # Create stub mock client and bucket MockClient.return_value MockBucket.return_value diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py index cc87c86e7af0b..2d39585cb1c5f 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py @@ -5,6 +5,7 @@ import pathlib import yaml + from metadata_service.models import transform from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py index 4946f69644846..b4ac1d44faadc 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py @@ -4,6 +4,7 @@ import requests import semver import yaml + from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.validators import metadata_validator diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py index 20afa5a6b391f..0948c73bdf39d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py @@ -11,6 +11,7 @@ from google.cloud import storage from orchestrator.logging import sentry + GROUP_NAME = "connector_metrics" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py index 6d2ca249f0760..1fe1ada3c5f18 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py @@ -22,6 +22,7 @@ ) from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + T = TypeVar("T") GROUP_NAME = "connector_test_report" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py index 63f9392442238..9d1d875feb18b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py @@ -17,6 +17,7 @@ from orchestrator.ops.slack import send_slack_message from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "github" TOOLING_TEAM_SLACK_TEAM_ID = "S077R8636CV" # We give 6 hours for the metadata to be updated diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py index ea3d0fb54a7d9..783f16614b44f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py @@ -16,6 +16,7 @@ from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition from orchestrator.utils.object_helpers import are_values_equal, merge_values + GROUP_NAME = "metadata" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py index dde8232c8a696..dc00e6412c8dd 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py @@ -20,6 +20,7 @@ from orchestrator.utils.object_helpers import default_none_to_dict from pydash.objects import set_with + PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] GROUP_NAME = "registry" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py index 657ef41ce5af0..d642f5179bda9 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py @@ -31,6 +31,7 @@ from pydantic import BaseModel, ValidationError from pydash.objects import get, set_with + GROUP_NAME = "registry_entry" # TYPES diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py index a1e26af19329e..a1fa7a6e198d9 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py @@ -22,6 +22,7 @@ ) from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "registry_reports" OSS_SUFFIX = "_oss" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py index 8bfae88677510..2bbcbd2834511 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py @@ -8,6 +8,7 @@ from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, Output, asset from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe + GROUP_NAME = "slack" USER_REQUEST_CHUNK_SIZE = 2000 diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py index a82cbc5df3553..3d52193b1b95b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py @@ -11,6 +11,7 @@ from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from orchestrator.logging import sentry + GROUP_NAME = "specs_secrets_mask" # HELPERS diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py index e94fa49d225cf..b7f803334efe9 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py @@ -5,6 +5,7 @@ import os from typing import Optional + DEFAULT_ASSET_URL = "https://storage.googleapis.com" VALID_REGISTRIES = ["oss", "cloud"] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py index 331ccab653d35..904963eeed03d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py @@ -7,6 +7,7 @@ import requests from orchestrator.models.metadata import LatestMetadataEntry + GROUP_NAME = "connector_cdk_versions" BASE_URL = "https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py index 7021e5a7d991e..f32a0a39fde4d 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py @@ -10,6 +10,7 @@ from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition + PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py index 4528a6d8c31f3..dd3eb607b43e7 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py @@ -4,6 +4,7 @@ from dagster import AssetSelection, define_asset_job + nightly_reports_inclusive = AssetSelection.keys("generate_nightly_report").upstream() generate_nightly_reports = define_asset_job(name="generate_nightly_reports", selection=nightly_reports_inclusive) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py index 01ce46dace71f..275f27e25813f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py @@ -4,6 +4,7 @@ from dagster import AssetSelection, define_asset_job + stale_gcs_latest_metadata_file_inclusive = AssetSelection.keys("stale_gcs_latest_metadata_file").upstream() generate_stale_gcs_latest_metadata_file = define_asset_job( name="generate_stale_metadata_report", selection=stale_gcs_latest_metadata_file_inclusive diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py index 322c2e5d3002a..3756f7ac8a87b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py @@ -7,6 +7,7 @@ from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus + oss_registry_inclusive = AssetSelection.keys("persisted_oss_registry", "specs_secrets_mask_yaml").upstream() generate_oss_registry = define_asset_job(name="generate_oss_registry", selection=oss_registry_inclusive) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py index e65d9fd691e8b..8e908183e80bc 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py @@ -8,6 +8,7 @@ import sentry_sdk from dagster import AssetExecutionContext, OpExecutionContext, SensorEvaluationContext, get_dagster_logger + sentry_logger = get_dagster_logger("sentry") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py index a8e791abd5bb7..1f956e619ead6 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Extra + # TODO (ben): When the pipeline project is brought into the airbyte-ci folder # we should update these models to import their twin models from the pipeline project diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py index 78477fe58639d..9eac1b19ebf19 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py @@ -13,6 +13,7 @@ from dagster._utils import mkdir_p from typing_extensions import TypeAlias + IOStream: TypeAlias = Union[TextIO, BinaryIO] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py index cc3a3d42f4023..abbb2aef30f6a 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py @@ -6,6 +6,7 @@ from dagster import DefaultSensorStatus, RunRequest, SensorDefinition, SensorEvaluationContext, SkipReason, build_resources, sensor + # e.g. 2023-06-02T17:42:36Z EXPECTED_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py index eaa76f7b30c7f..1fcbade3c6a4e 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py @@ -12,6 +12,7 @@ from jinja2 import Environment, PackageLoader from orchestrator.utils.object_helpers import deep_copy_params + # 🔗 HTML Renderers diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py index 64b482884b697..b5c996fe26c00 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py @@ -8,6 +8,7 @@ import pandas as pd from dagster import MetadataValue, Output + OutputDataFrame = Output[pd.DataFrame] CURSOR_SEPARATOR = ":" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py index 19b57416297a4..77ebca0a4ee3c 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py @@ -9,6 +9,7 @@ import mergedeep from deepdiff import DeepDiff + T = TypeVar("T") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py index 27088881908ef..b0a7f223af3a3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger + from pipelines import main_logger from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py index e26dc095a7c08..a6565dfe54955 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py @@ -11,6 +11,7 @@ from click import UsageError from connector_ops.utils import Connector # type: ignore from dagger import Container, ExecError, Platform, QueryError + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.utils import export_container_to_tarball, sh_dash_c from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py index 8d31bd5a714a7..70bcdfaf18377 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py @@ -3,6 +3,7 @@ # from dagger import Container, Directory, File, Platform, QueryError + from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.steps.gradle import GradleTask diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py index 0c9116d592da0..72c9dc386f1e0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py @@ -6,13 +6,14 @@ from typing import Any from dagger import Container, Platform +from pydash.objects import get # type: ignore + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import COMPONENTS_FILE_PATH, MANIFEST_FILE_PATH from pipelines.dagger.actions.python.common import apply_python_development_overrides from pipelines.models.steps import StepResult -from pydash.objects import get # type: ignore class BuildConnectorImages(BuildConnectorImagesBase): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py index 3eaa9e9b088dc..2cfe294a0cacd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py @@ -3,6 +3,7 @@ # from dagger import Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.connector import normalization from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py index 262fdbc780d34..802eae40cca04 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -6,6 +6,7 @@ from typing import Any from dagger import Container, Platform + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py index 3c7ab35dcff5a..22dee605124b2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import semver + from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index ca69acb7a4a06..6de4dc1e0260f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -8,6 +8,7 @@ import asyncclick as click from connector_ops.utils import Connector, ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore + from pipelines import main_logger from pipelines.cli.airbyte_ci import wrap_in_secret from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 52deb09d7c013..ecb901945486a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -15,6 +15,7 @@ from asyncer import asyncify from dagger import Directory, Platform from github import PullRequest + from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import BUILD_PLATFORMS from pipelines.dagger.actions import secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py index 156b7ab732a2c..6203febb51b9b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/commands.py @@ -4,6 +4,7 @@ from typing import List import asyncclick as click + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.generate_erd.pipeline import run_connector_generate_erd_pipeline from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py index 54b0cd4b8e5e3..17b70bd2475cf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, List from dagger import Container, Directory + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext @@ -51,7 +52,7 @@ async def _run(self, connector_to_discover: Container) -> StepResult: command.append("--skip-llm-relationships") erd_directory = self._build_erd_container(connector_directory, discovered_catalog).with_exec(command).directory("/source/erd") - await (erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory)))) + await erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory))) return StepResult(step=self, status=StepStatus.SUCCESS, output=erd_directory) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 72bfadc2cc203..dea261788ba5c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -7,10 +7,11 @@ import asyncclick as click from connector_ops.utils import console # type: ignore -from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from rich.table import Table from rich.text import Text +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + @click.command(cls=DaggerPipelineCommand, help="List all selected connectors.", name="list") @click.option( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py index 4c933ce975ec0..d87831d845351 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_base_image.pipeline import run_connector_migration_to_base_image_pipeline from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py index 62d62afcb7170..57ea1b10d8cf7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py @@ -8,6 +8,7 @@ from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Directory from jinja2 import Template + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata @@ -15,7 +16,6 @@ from pipelines.models.steps import Step, StepResult, StepStatus if TYPE_CHECKING: - from anyio import Semaphore @@ -89,7 +89,6 @@ async def _run(self) -> StepResult: ) def add_build_instructions(self, og_doc_content: str) -> str: - build_instructions_template = Template( textwrap.dedent( """ diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py index 2c5070adc84e1..da73b3ebc0228 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.migrate_to_inline_schemas.pipeline import run_connector_migrate_to_inline_schemas_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py index ccd85b81adbd4..ba880ae73840a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, List from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines import main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py index 2270baa15b3dc..9d32a6e976147 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.migrate_to_logging_logger.pipeline import run_connector_migrate_to_logging_logger_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py index 1e8d5ce8500ab..d79eedaaa7c38 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_logging_logger/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py index 07ac847dd7d3e..f7440de7479a8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_manifest_only.pipeline import run_connectors_manifest_only_pipeline from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines @@ -12,7 +13,6 @@ @click.command(cls=DaggerPipelineCommand, short_help="Migrate a low-code connector to manifest-only") @click.pass_context async def migrate_to_manifest_only(ctx: click.Context) -> bool: - connectors_contexts = [ ConnectorContext( pipeline_name=f"Migrate connector {connector.technical_name} to manifest-only", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py index e32946d260871..89e167de379ee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py @@ -10,6 +10,7 @@ import git # type: ignore from anyio import Semaphore # type: ignore from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_manifest_only.manifest_component_transformer import ManifestComponentTransformer @@ -297,7 +298,6 @@ async def _run(self) -> StepResult: ## MAIN FUNCTION ## async def run_connectors_manifest_only_pipeline(context: ConnectorContext, semaphore: "Semaphore", *args: Any) -> Report: - steps_to_run: STEP_TREE = [] steps_to_run.append([StepToRun(id=CONNECTOR_TEST_STEP_ID.MANIFEST_ONLY_CHECK, step=CheckIsManifestMigrationCandidate(context))]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py index 4e73eda537a03..eef7b8c9e1cf4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.migrate_to_poetry.pipeline import run_connector_migration_to_poetry_pipeline_wrapper from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines @@ -31,7 +32,6 @@ ) @click.pass_context async def migrate_to_poetry(ctx: click.Context, changelog: bool, bump: str | None) -> bool: - connectors_contexts = [ ConnectorContext( pipeline_name=f"Migrate {connector.technical_name} to Poetry", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py index ccb576c47c9d8..f29bfb213fffb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/pipeline.py @@ -12,6 +12,7 @@ import toml from connector_ops.utils import ConnectorLanguage # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py index 4e6f03512e18b..2ca0f8c55b083 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py @@ -3,6 +3,7 @@ # """This module groups the functions to run full pipelines for connector testing.""" + from __future__ import annotations import sys @@ -13,6 +14,7 @@ import dagger from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Config + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 5a93fa05b2f97..67df171d49b3c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, Iterable, List import asyncclick as click + from pipelines import main_logger from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext, RolloutMode diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index f4e3e132eccc3..702461da73dc7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -9,6 +9,7 @@ import asyncclick as click from github import PullRequest + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import PUBLISH_FAILURE_SLACK_CHANNEL, PUBLISH_UPDATES_SLACK_CHANNEL, ContextState from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles @@ -136,7 +137,6 @@ def get_slack_channels(self) -> List[str]: return [PUBLISH_UPDATES_SLACK_CHANNEL] def create_slack_message(self) -> str: - docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" message = f"*{self.rollout_mode.value} <{docker_hub_url}|{self.docker_image}>*\n" if self.is_ci: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 56af45b38413b..4ced8a931b463 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -15,6 +15,8 @@ from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification # type: ignore from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage # type: ignore from dagger import Container, Directory, ExecError, File, ImageLayerCompression, Platform, QueryError +from pydantic import BaseModel, ValidationError + from pipelines import consts from pipelines.airbyte_ci.connectors.build_image import steps from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext, RolloutMode @@ -30,7 +32,6 @@ from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file from pipelines.helpers.pip import is_package_published from pipelines.models.steps import Step, StepModifyingFiles, StepResult, StepStatus -from pydantic import BaseModel, ValidationError class InvalidSpecOutputError(Exception): @@ -244,8 +245,10 @@ async def check_if_image_only_has_gzip_layers(self) -> bool: async def _run(self, attempt: int = 3) -> StepResult: try: try: - await self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image}").with_exec( - ["spec"], use_entrypoint=True + await ( + self.context.dagger_client.container() + .from_(f"docker.io/{self.context.docker_image}") + .with_exec(["spec"], use_entrypoint=True) ) except ExecError: if attempt > 0: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py index daf63427e58b0..6237ffdc17863 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pull_request/commands.py @@ -4,6 +4,7 @@ import asyncclick as click + from pipelines.airbyte_ci.connectors.pull_request.pipeline import run_connector_pull_request_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 68f75275ad03e..1447a10fcee8c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -12,24 +12,26 @@ from connector_ops.utils import console # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape +from rich.console import Group +from rich.panel import Panel +from rich.style import Style +from rich.table import Table +from rich.text import Text + from pipelines.consts import GCS_PUBLIC_DOMAIN from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX, AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX from pipelines.helpers.utils import format_duration from pipelines.models.artifacts import Artifact from pipelines.models.reports import Report from pipelines.models.steps import StepStatus -from rich.console import Group -from rich.panel import Panel -from rich.style import Style -from rich.table import Table -from rich.text import Text if TYPE_CHECKING: from typing import List - from pipelines.airbyte_ci.connectors.context import ConnectorContext from rich.tree import RenderableType + from pipelines.airbyte_ci.connectors.context import ConnectorContext + @dataclass(frozen=True) class ConnectorReport(Report): @@ -133,9 +135,9 @@ def to_html(self) -> str: template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url - template_context[ - "icon_url" - ] = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + template_context["icon_url"] = ( + f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + ) return template.render(template_context) async def save_html_report(self) -> None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 442a5a8242453..7a4b1179edd96 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -6,6 +6,7 @@ from typing import Dict, List import asyncclick as click + from pipelines import main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py index 2b94b34e7a504..b941d05f034ec 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/context.py @@ -8,11 +8,12 @@ from logging import Logger from typing import Any, Dict, List, Optional +from pydash import find # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.execution.run_steps import RunStepOptions from pipelines.models.secrets import Secret, SecretNotFoundError, SecretStore -from pydash import find # type: ignore # These test suite names are declared in metadata.yaml files TEST_SUITE_NAME_TO_STEP_ID = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py index 16533aa192ddf..495bfc17d5701 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -9,6 +9,7 @@ import anyio from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext @@ -17,7 +18,6 @@ from pipelines.helpers.execution.run_steps import StepToRun, run_steps if TYPE_CHECKING: - from pipelines.helpers.execution.run_steps import STEP_TREE LANGUAGE_MAPPING = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index afecc5fe00ee6..7fbd7c8a063f5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -20,6 +20,11 @@ import semver import yaml # type: ignore from dagger import Container, Directory + +# This slugify lib has to be consistent with the slugify lib used in live_tests +# live_test can't resolve the passed connector container otherwise. +from slugify import slugify # type: ignore + from pipelines import hacks, main_logger from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -33,10 +38,6 @@ from pipelines.models.secrets import Secret from pipelines.models.steps import STEP_PARAMS, MountPath, Step, StepResult, StepStatus -# This slugify lib has to be consistent with the slugify lib used in live_tests -# live_test can't resolve the passed connector container otherwise. -from slugify import slugify # type: ignore - GITHUB_URL_PREFIX_FOR_CONNECTORS = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/master/airbyte-integrations/connectors" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py index 2cf011a19d65f..caef578365705 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py @@ -3,12 +3,14 @@ # """This module groups steps made to run tests for a specific Java connector given a test context.""" + from __future__ import annotations from typing import TYPE_CHECKING import anyio from dagger import File, QueryError + from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import ( BuildConnectorDistributionTar, BuildConnectorImages, @@ -89,7 +91,6 @@ def _create_integration_step_args_factory(context: ConnectorTestContext) -> Call """ async def _create_integration_step_args(results: RESULTS_DICT) -> Dict[str, Optional[File]]: - connector_container = results["build"].output[LOCAL_BUILD_PLATFORM] connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container, LOCAL_BUILD_PLATFORM) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py index c41ed2652a28c..00a5eca6be572 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/manifest_only_connectors.py @@ -7,6 +7,7 @@ from typing import List, Sequence, Tuple from dagger import Container, File + from pipelines.airbyte_ci.connectors.build_image.steps.manifest_only_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.test.context import ConnectorTestContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 5f38935d67e41..a967bcaefdf10 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -8,9 +8,10 @@ from typing import List, Sequence, Tuple import dpath.util +from dagger import Container, File + import pipelines.dagger.actions.python.common import pipelines.dagger.actions.system.docker -from dagger import Container, File from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py index 5bce0e3d597da..03111abf2ef1d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py @@ -5,6 +5,7 @@ from typing import List import asyncclick as click + from pipelines.airbyte_ci.connectors.up_to_date.pipeline import run_connector_up_to_date_pipeline from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.connectors.command import run_connector_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py index bd9dd2b6ea954..b8fea724e5e72 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/pipeline.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING from jinja2 import Environment, PackageLoader, select_autoescape + from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -25,6 +26,7 @@ from anyio import Semaphore from github import PullRequest + from pipelines.models.steps import StepResult UP_TO_DATE_PR_LABEL = "up-to-date" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py index 457b964849ec5..f461fb8d0c887 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/steps.py @@ -14,6 +14,7 @@ import dagger from connector_ops.utils import POETRY_LOCK_FILE_NAME, PYPROJECT_FILE_NAME # type: ignore from deepdiff import DeepDiff # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.models.steps import Step, StepModifyingFiles, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py index 35ce460599bfa..16519b4e6d6e9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py index 623624f2f296b..a764dcef2bccb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py @@ -10,6 +10,7 @@ import toml from connector_ops.utils import ConnectorLanguage # type: ignore from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import LOCAL_BUILD_PLATFORM diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index a1f2d3cc613f5..c1e569392db1b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -5,6 +5,7 @@ """ Module exposing the format commands. """ + from __future__ import annotations import logging @@ -12,6 +13,7 @@ from typing import Dict, List import asyncclick as click + from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index b18464b1539fe..447acc8be71b9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Union import dagger + from pipelines.airbyte_ci.format.consts import CACHE_MOUNT_PATH, DEFAULT_FORMAT_IGNORE_LIST, REPO_MOUNT_PATH, WARM_UP_INCLUSIONS, Formatter from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME, GO_IMAGE, MAVEN_IMAGE, NODE_IMAGE, PYTHON_3_10_IMAGE from pipelines.helpers import cache_keys diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py index da6e4054270f3..195eb7639ec5e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py @@ -9,6 +9,7 @@ import asyncclick as click import dagger + from pipelines import main_logger from pipelines.airbyte_ci.format.actions import list_files_in_directory from pipelines.airbyte_ci.format.configuration import Formatter diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py index a140b322309f8..98b801eb8e27d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py @@ -3,6 +3,7 @@ # import asyncclick as click + from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py index de33dfcb254a1..f8a4d490ff204 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py @@ -8,6 +8,7 @@ import asyncclick as click import dagger + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.steps.docker import SimpleDockerStep diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 72dbe53b170f8..e40905692072c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -5,9 +5,11 @@ """ Module exposing the format commands. """ + from __future__ import annotations import asyncclick as click + from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index e02c853a4275b..d822ae8e890bd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -5,12 +5,14 @@ """ Module exposing the format commands. """ + from __future__ import annotations from typing import Optional import asyncclick as click from packaging import version + from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py index 6a2fcc020b321..210cd2f8bc5f6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/base_image.py @@ -7,6 +7,7 @@ import yaml from base_images import version_registry # type: ignore from connector_ops.utils import METADATA_FILE_NAME # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file from pipelines.models.steps import StepModifyingFiles, StepResult, StepStatus @@ -22,7 +23,6 @@ class NoBaseImageAddressInMetadataError(Exception): class UpdateBaseImageMetadata(StepModifyingFiles): - BASE_IMAGE_LIST_CACHE_TTL_SECONDS = 60 * 60 * 24 # 1 day context: ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py index 3bf22b823a95f..e26c10c292c10 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/bump_version.py @@ -9,6 +9,7 @@ import semver import yaml # type: ignore from connector_ops.utils import METADATA_FILE_NAME, PYPROJECT_FILE_NAME # type: ignore + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py index 96386bf0f5370..3170d082b3337 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/changelog.py @@ -8,6 +8,7 @@ import semver from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.helpers.changelog import Changelog from pipelines.helpers.connectors.dagger_fs import dagger_read_file, dagger_write_file @@ -33,7 +34,6 @@ def __init__( self.pull_request_number = pull_request_number or "*PR_NUMBER_PLACEHOLDER*" async def _run(self, pull_request_number: int | str | None = None) -> StepResult: - if pull_request_number is None: # this allows passing it dynamically from a result of another action (like creating a pull request) pull_request_number = self.pull_request_number diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py index 191687f0ac19f..1e974fc46deb7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -5,6 +5,7 @@ from typing import Dict, List, Optional import dagger + from pipelines.dagger.actions.python.pipx import with_installed_pipx_package from pipelines.dagger.containers.python import with_python_base from pipelines.models.contexts.pipeline_context import PipelineContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 0cc44ad511af2..8dfdb42bbbd72 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -6,9 +6,10 @@ from datetime import datetime from typing import Any, ClassVar, List, Optional, Tuple, cast -import pipelines.dagger.actions.system.docker import requests from dagger import CacheSharingMode, CacheVolume, Container, ExecError + +import pipelines.dagger.actions.system.docker from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME, AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py index 17c5a3312c08c..65c31518ccf15 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -11,6 +11,7 @@ import tomli import tomli_w from dagger import Container, Directory + from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.utils import sh_dash_c diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 117a1cf5e0d01..15ec6831423ba 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import asyncer + from pipelines.airbyte_ci.test import INTERNAL_POETRY_PACKAGES, INTERNAL_POETRY_PACKAGES_PATH, pipeline from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.git import get_modified_files diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py index 7f1670636ccba..3b7b758b5120c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py @@ -9,6 +9,7 @@ import asyncer import dagger import toml + from pipelines.airbyte_ci.test.models import deserialize_airbyte_ci_config from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_VERSION, POETRY_CACHE_VOLUME_NAME, PYPROJECT_TOML_FILE_PATH from pipelines.dagger.actions.system import docker diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py index c633f59db1d22..690c5ae56f94c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py @@ -5,6 +5,7 @@ import logging import asyncclick as click + from pipelines.cli.auto_update import is_dev_command from pipelines.external_scripts.airbyte_ci_dev_install import main as install_airbyte_ci_dev_pipx from pipelines.external_scripts.airbyte_ci_install import main as install_airbyte_ci_binary diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index fb7222ed19299..a9335d2cab3f2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -22,6 +22,7 @@ import asyncclick as click import docker # type: ignore from github import PullRequest + from pipelines import main_logger from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag from pipelines.cli.click_decorators import ( diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py index a97291aafaf94..6f7c16bae0cad 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py @@ -13,6 +13,7 @@ import asyncclick as click import requests + from pipelines import main_logger from pipelines.cli.confirm_prompt import confirm from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index b88f582c6e37b..7ede57342b7b4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -8,6 +8,7 @@ from typing import Any, Callable, Type, TypeVar import asyncclick as click + from pipelines.models.ci_requirements import CIRequirements _AnyCallable = Callable[..., Any] diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py index dfc0bd9d626f5..9120bf59921ee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py @@ -3,6 +3,7 @@ # """This module groups util function used in pipelines.""" + from __future__ import annotations import sys @@ -10,6 +11,7 @@ import asyncclick as click from dagger import DaggerError + from pipelines import consts, main_logger from pipelines.consts import GCS_PUBLIC_DOMAIN, STATIC_REPORT_PREFIX from pipelines.helpers import sentry_utils diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py index 5233cd62bef85..184d4ada00d6d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -14,6 +14,7 @@ import pkg_resources # type: ignore import requests # type: ignore + from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME LOGGER = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py index 84923558d3d14..a477e89d2a71c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/secrets.py @@ -3,6 +3,7 @@ from typing import Any, Optional import asyncclick as click + from pipelines.helpers.gcs import sanitize_gcp_credentials from pipelines.models.secrets import InMemorySecretStore, Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py index 4223f86b26ec8..c9e506813c04c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py @@ -7,6 +7,7 @@ from importlib.abc import Loader from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py index 9fe2806b7e279..406293ab94372 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py @@ -5,6 +5,7 @@ from typing import Any, Dict from dagger import Container, Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 22e11f22619b0..5e5a3808b7b3d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -8,6 +8,7 @@ from click import UsageError from dagger import Container, Directory + from pipelines import hacks from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import PATH_TO_LOCAL_CDK diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py index 3c16cb043520a..aa8b7cd90654b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py @@ -5,6 +5,7 @@ from typing import List, Optional from dagger import Container + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package from pipelines.dagger.actions.python.poetry import find_local_dependencies_in_pyproject_toml diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py index 09bf5f683462c..cc87e2506734b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py @@ -8,6 +8,7 @@ import toml from dagger import Container, Directory + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import AIRBYTE_SUBMODULE_DIR_NAME from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py index b7f9095b95929..f8f8f28accf72 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py @@ -9,6 +9,7 @@ from typing import List, Optional, Tuple from dagger import Client, File + from pipelines.helpers.utils import get_exec_result, secret_host_variable, with_exit_code from pipelines.models.secrets import Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py index 81773fd7ea4a5..606adce68c8f0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -3,6 +3,7 @@ # """This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory.""" + from __future__ import annotations from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ from typing import Callable, List from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py index 61ec8c4011064..72f1827c9939b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py @@ -6,9 +6,9 @@ import uuid from typing import Callable, Dict, List, Optional, Union -from dagger import Client, Container, File +from dagger import Client, Container, File, Service from dagger import Secret as DaggerSecret -from dagger import Service + from pipelines import consts from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import ( @@ -229,9 +229,8 @@ def with_crane( if context.docker_hub_username and context.docker_hub_password: base_container = ( - base_container.with_secret_variable( - "DOCKER_HUB_USERNAME", context.docker_hub_username.as_dagger_secret(context.dagger_client) - ).with_secret_variable("DOCKER_HUB_PASSWORD", context.docker_hub_password.as_dagger_secret(context.dagger_client)) + base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username.as_dagger_secret(context.dagger_client)) + .with_secret_variable("DOCKER_HUB_PASSWORD", context.docker_hub_password.as_dagger_secret(context.dagger_client)) # We use sh -c to be able to use environment variables in the command # This is a workaround as the default crane entrypoint doesn't support environment variables .with_exec(sh_dash_c(["crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"])) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py index cf38d65be91b3..790063e2ce660 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py @@ -4,6 +4,7 @@ from typing import Optional from dagger import Client, Container + from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL from pipelines.helpers.utils import sh_dash_c diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py index 2eb4241894391..8438fcec56e10 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py @@ -3,6 +3,7 @@ # from dagger import Container, Secret + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import INTERNAL_TOOL_PATHS from pipelines.dagger.actions.python.pipx import with_installed_pipx_package diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py index 47bbe7822b214..9c595db41f633 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -5,6 +5,7 @@ import datetime from dagger import CacheVolume, Container, File, Platform + from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions.connector.hooks import finalize_build diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 245190f66ede4..7bf7afc8521ee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -4,6 +4,7 @@ from dagger import CacheSharingMode, CacheVolume, Client, Container + from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import ( CONNECTOR_TESTING_REQUIREMENTS, diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 76c9682369474..28b7b298c4ae0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -11,12 +11,14 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage # type: ignore + from pipelines import consts from pipelines.airbyte_ci.steps.base_image import UpdateBaseImageMetadata from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, is_automerge_pull_request, update_commit_status_check if TYPE_CHECKING: from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.models.steps import StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py index 41a87b27cd11b..a062b185f767e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/changelog.py @@ -9,6 +9,7 @@ from typing import Set, Tuple import semver + from pipelines.helpers.github import AIRBYTE_GITHUB_REPO diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index 4f601b7e83dc7..4440fd4e466c3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -10,6 +10,7 @@ import asyncclick as click import asyncer from jinja2 import Template + from pipelines.models.steps import CommandResult ALL_RESULTS_KEY = "_run_all_results" diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py index ad9531e907028..2f822c0e8f2bf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/command.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Callable, List import asyncclick as click + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py index 98c2d95bb69c1..a6e8102bdf356 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py @@ -7,6 +7,7 @@ from typing import FrozenSet, Set, Union from connector_ops.utils import Connector # type: ignore + from pipelines import main_logger from pipelines.helpers.utils import IGNORED_FILE_EXTENSIONS, METADATA_FILE_NAME diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py index 7330f99039fc2..2d6abeede1adb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py @@ -13,6 +13,7 @@ import anyio import asyncer import dpath + from pipelines import main_logger from pipelines.models.steps import StepStatus @@ -191,7 +192,6 @@ def _filter_skipped_steps(steps_to_evaluate: STEP_TREE, skip_steps: List[str], r """ steps_to_run: STEP_TREE = [] for step_to_eval in steps_to_evaluate: - # ignore nested steps if isinstance(step_to_eval, list): steps_to_run.append(step_to_eval) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py index 0917f035ea6cd..a73cf7f686ec4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py @@ -8,6 +8,7 @@ from google.cloud import storage # type: ignore from google.oauth2 import service_account # type: ignore + from pipelines import main_logger from pipelines.consts import GCS_PUBLIC_DOMAIN diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py index 9ef656747a5e9..926a07a768672 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -7,6 +7,7 @@ import git from dagger import Connection, SessionError + from pipelines.consts import CIContext from pipelines.dagger.containers.git import checked_out_git_container from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index ac48aaa8625a7..914ecd8c3d647 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -14,6 +14,7 @@ import github as github_sdk from connector_ops.utils import console # type: ignore + from pipelines import main_logger from pipelines.consts import CIContext from pipelines.models.secrets import Secret @@ -107,7 +108,6 @@ def get_pull_request(pull_request_number: int, github_access_token: Secret) -> g def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Optional[Logger] = None) -> None: - update_commit_status_check( click_context["git_revision"], github_state, @@ -138,7 +138,6 @@ def create_or_update_github_pull_request( labels: Optional[Iterable[str]] = None, force_push: bool = True, ) -> github_sdk.PullRequest.PullRequest: - logger = logger or main_logger g = github_sdk.Github(auth=github_sdk.Auth.Token(github_token)) repo = g.get_repo(repo_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py index 28256cc78789c..ae190629e3642 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, Optional from asyncclick import Command, Context + from pipelines.models.steps import Step diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index 12ae528067870..38c1818b0b7b1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -7,6 +7,7 @@ import typing import requests + from pipelines import main_logger if typing.TYPE_CHECKING: diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index ca397153becda..6160924f7a785 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -3,6 +3,7 @@ # """This module groups util function used in pipelines.""" + from __future__ import annotations import contextlib diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py index 4f8776f525fe6..295faf9acbcb4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/artifacts.py @@ -5,6 +5,7 @@ from typing import Optional import dagger + from pipelines.consts import GCS_PUBLIC_DOMAIN from pipelines.dagger.actions import remote_storage from pipelines.models.secrets import Secret diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index a4c3f623aa53f..471b5840addb2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -11,9 +11,10 @@ import anyio import dagger from asyncclick import Context, get_current_context +from pydantic import BaseModel, Field, PrivateAttr + from pipelines import main_logger from pipelines.cli.click_decorators import LazyPassDecorator -from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton @@ -86,7 +87,6 @@ async def get_dagger_client(self) -> dagger.Client: if not self._dagger_client: async with self._dagger_client_lock: if not self._dagger_client: - connection = dagger.Connection(dagger.Config(log_output=self.get_log_output())) """ Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py index 323599cc9210d..8b816c31334ec 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -15,10 +15,10 @@ from typing import TYPE_CHECKING, Dict from asyncer import asyncify -from dagger import Client, Directory, File, GitRepository +from dagger import Client, Directory, File, GitRepository, Service from dagger import Secret as DaggerSecret -from dagger import Service from github import PullRequest + from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import MANUAL_PIPELINE_STATUS_CHECK_OVERRIDE_PREFIXES, CIContext, ContextState from pipelines.helpers.execution.run_steps import RunStepOptions @@ -343,7 +343,9 @@ async def __aexit__( if self.should_send_slack_message: # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook await asyncify(send_message_to_webhook)( - self.create_slack_message(), self.get_slack_channels(), self.slack_webhook # type: ignore + self.create_slack_message(), + self.get_slack_channels(), + self.slack_webhook, # type: ignore ) # supress the exception if it was handled return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py index 5d884398653fb..408b5ecded6b5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py @@ -15,20 +15,22 @@ from typing import List from connector_ops.utils import console # type: ignore -from pipelines.consts import LOCAL_REPORTS_PATH_ROOT -from pipelines.helpers.utils import format_duration, slugify -from pipelines.models.artifacts import Artifact -from pipelines.models.steps import StepResult, StepStatus from rich.console import Group from rich.panel import Panel from rich.style import Style from rich.table import Table from rich.text import Text +from pipelines.consts import LOCAL_REPORTS_PATH_ROOT +from pipelines.helpers.utils import format_duration, slugify +from pipelines.models.artifacts import Artifact +from pipelines.models.steps import StepResult, StepStatus + if typing.TYPE_CHECKING: - from pipelines.models.contexts.pipeline_context import PipelineContext from rich.tree import RenderableType + from pipelines.models.contexts.pipeline_context import PipelineContext + @dataclass(frozen=True) class Report: diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index d46ad47d737d8..5f3f5c2bd70bb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -16,6 +16,7 @@ import asyncer import click from dagger import Client, Container, DaggerError + from pipelines import main_logger from pipelines.helpers import sentry_utils from pipelines.helpers.utils import format_duration, get_exec_result @@ -23,8 +24,10 @@ from pipelines.models.secrets import Secret if TYPE_CHECKING: + from typing import Any, ClassVar, Optional, Set, Union + import dagger - from typing import Any, ClassVar, Optional, Union, Set + from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.models.contexts.pipeline_context import PipelineContext @@ -120,7 +123,6 @@ def __str__(self) -> str: # noqa D105 @dataclass(kw_only=True, frozen=True) class PoeTaskResult(Result): - task_name: str def __repr__(self) -> str: # noqa D105 @@ -418,7 +420,6 @@ def _get_timed_out_step_result(self) -> StepResult: class StepModifyingFiles(Step): - modified_files: List[str] modified_directory: dagger.Directory diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index fe7314bbebe7a..9226b46b7072a 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -13,6 +13,7 @@ import pytest import requests from connector_ops.utils import Connector + from pipelines.helpers import utils from tests.utils import ALL_CONNECTORS diff --git a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py index 17f1d746abf31..691de4de1aa5a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py +++ b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py @@ -4,6 +4,7 @@ import pytest from click import UsageError + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python import common from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles @@ -37,8 +38,11 @@ async def test_apply_python_development_overrides( mocker.patch.object(common, "PATH_TO_LOCAL_CDK", local_cdk_path) if local_cdk_is_available: local_cdk_path.mkdir() - await dagger_client.git("https://github.com/airbytehq/airbyte-python-cdk", keep_git_dir=False).branch("main").tree().export( - str(local_cdk_path) + await ( + dagger_client.git("https://github.com/airbytehq/airbyte-python-cdk", keep_git_dir=False) + .branch("main") + .tree() + .export(str(local_cdk_path)) ) connector_context.use_local_cdk = use_local_cdk fake_connector_container = connector_context.dagger_client.container().from_("airbyte/python-connector-base:3.0.0") diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py index 15808f2c88de2..5f6045942f35c 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_bases.py +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -7,6 +7,7 @@ import anyio import pytest from dagger import DaggerError + from pipelines.models import reports, steps pytestmark = [ diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py index 7b8d288825842..13b2e59d6db46 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_manifest_only_connectors.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, manifest_only_connectors from pipelines.consts import BUILD_PLATFORMS from pipelines.models.steps import StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py index 7b17632d5c48d..f03496b7d99a4 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py @@ -6,6 +6,7 @@ import asyncclick as click import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import BUILD_PLATFORMS diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py index ed0cfe865f35e..553dbcea6579d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py @@ -7,6 +7,7 @@ import dagger import docker import pytest + from pipelines.airbyte_ci.connectors.build_image.steps import common from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.models.steps import StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_changelog.py b/airbyte-ci/connectors/pipelines/tests/test_changelog.py index 1ad9bdfe1684c..4261048276b14 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_changelog.py +++ b/airbyte-ci/connectors/pipelines/tests/test_changelog.py @@ -8,6 +8,7 @@ import pytest import semver + from pipelines.helpers.changelog import Changelog, ChangelogParsingException pytestmark = [ diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index cf503c9d65881..3aa7ecfeecdb1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -5,6 +5,7 @@ import asyncclick as click import pytest from asyncclick.testing import CliRunner + from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index 3b33ce91a425e..b3f3b20577127 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -8,6 +8,7 @@ import pytest from asyncclick.testing import CliRunner from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage + from pipelines.airbyte_ci.connectors import commands as connectors_commands from pipelines.airbyte_ci.connectors.build_image import commands as connectors_build_command from pipelines.airbyte_ci.connectors.publish import commands as connectors_publish_command diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py index f504b27654ae6..2cec51a3ee3ff 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -6,6 +6,7 @@ import asyncclick as click import pytest import requests + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.dagger.actions.python import common @@ -63,7 +64,6 @@ def python_connector_base_image_address(python_connector_with_setup_not_latest_c async def test_with_python_connector_installed_from_setup(context_with_setup, python_connector_base_image_address, latest_cdk_version): - python_container = context_with_setup.dagger_client.container().from_(python_connector_base_image_address) user = await BuildConnectorImages.get_image_user(python_container) container = await common.with_python_connector_installed( diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py index 61ddb94661560..a3c33327516a2 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py +++ b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py @@ -6,6 +6,7 @@ import pytest from asyncclick.testing import CliRunner + from pipelines.airbyte_ci.format import commands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index 96a397e728554..e0f3604b5448e 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -5,8 +5,9 @@ from pathlib import Path -import pipelines.helpers.connectors.modifed import pytest + +import pipelines.helpers.connectors.modifed from pipelines.airbyte_ci.steps import gradle from pipelines.models import steps diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py index 7201a2b83059c..a8630cb7988bc 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py @@ -5,6 +5,7 @@ import anyio import pytest + from pipelines.helpers.execution import argument_parsing diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py index 30dcc42e91438..069702081d39d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py @@ -4,6 +4,7 @@ import anyio import pytest + from pipelines.helpers.execution.run_steps import InvalidStepConfiguration, RunStepOptions, StepToRun, run_steps from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py index 26605c6758497..fe1e881960607 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_pip.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pytest + from pipelines.helpers.pip import is_package_published diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py index d700a54fe7ad6..d13c3ca154933 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py @@ -8,6 +8,7 @@ import dagger import pytest from connector_ops.utils import Connector, ConnectorLanguage + from pipelines import consts from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers import utils @@ -225,7 +226,6 @@ async def test_export_container_to_tarball(mocker, dagger_client, tmp_path, tar_ @pytest.mark.anyio async def test_export_container_to_tarball_failure(mocker, tmp_path): - context = mocker.Mock( connector=mocker.Mock(technical_name="my_connector"), host_image_export_dir_path=tmp_path, diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index dd12b88a930e4..3a554f3435cec 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -5,6 +5,7 @@ import asyncclick as click import dagger import pytest + from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py index d3d2f41123ac8..65e46ef028fb1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -7,6 +7,7 @@ import pytest import requests from dagger import Client, Platform + from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 57cf9645338e0..3c7f1613faa5f 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -8,6 +8,7 @@ import anyio import pytest + from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.airbyte_ci.connectors.publish.context import RolloutMode from pipelines.models.steps import StepStatus @@ -361,7 +362,6 @@ async def test_run_connector_python_registry_publish_pipeline( expect_build_connector_called, api_token, ): - for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py index b6b1598a75d94..9671d3e3d648e 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest + from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.helpers.utils import get_exec_result from pipelines.models.contexts.pipeline_context import PipelineContext diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py index 3475fc1d1d9c0..02c5276ce4787 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_version_check.py @@ -2,9 +2,10 @@ import pytest from connector_ops.utils import METADATA_FILE_NAME -from pipelines.airbyte_ci.connectors.test.steps.common import VersionIncrementCheck from semver import VersionInfo +from pipelines.airbyte_ci.connectors.test.steps.common import VersionIncrementCheck + class TestVersionIncrementCheck: @pytest.fixture @@ -16,7 +17,6 @@ def context(self, mocker, tmp_path): return context def _get_version_increment_check(self, mocker, context, master_version="1.0.0", current_version="1.0.1"): - mocker.patch( "pipelines.airbyte_ci.connectors.test.steps.common.VersionIncrementCheck.master_connector_version", new_callable=mocker.PropertyMock, diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 90efcfd23f927..4019951179a9f 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -12,6 +12,7 @@ import pytest import yaml from freezegun import freeze_time + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps import common from pipelines.dagger.actions.system import docker diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index 5228e34cd28c8..7980768e586da 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -7,6 +7,7 @@ import asyncclick as click import pytest from connector_ops.utils import Connector, ConnectorLanguage + from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps.python_connectors import PyAirbyteValidation, UnitTests diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py index 3beaf70d65a39..4e515605107dd 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_java_cdk.py @@ -13,6 +13,7 @@ import pytest from connector_ops.utils import Connector, ConnectorLanguage from dagger import Directory + from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py index 8f407c30a8c2c..9c0047ac4c3a3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/base.py @@ -5,6 +5,7 @@ import inflection import pytest + from connector_acceptance_test.config import Config diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index a29dd584c1a3e..6f2a376b73a22 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel + config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration") invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration") spec_path: str = Field( diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 1f29cdaac846b..357f38318df9f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -17,6 +17,7 @@ import dagger import pytest + from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import ( diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py index 82d32a4aaf520..23665c66165eb 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/plugin.py @@ -10,11 +10,13 @@ import pytest from _pytest.config import Config from _pytest.config.argparsing import Parser + from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import Config as AcceptanceTestConfig from connector_acceptance_test.config import GenericTestConfig from connector_acceptance_test.utils import diff_dicts, load_config + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py index df276b655fd7c..4b6ed43bfb061 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py @@ -9,9 +9,11 @@ from typing import Any, Dict, List, Mapping import pendulum -from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog from jsonschema import Draft7Validator, FormatChecker, FormatError, ValidationError, validators +from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog + + # fmt: off timestamp_regex = re.compile((r"^\d{4}-\d?\d-\d?\d" # date r"(\s|T)" # separator diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py index e91ee1935415c..7b1365f38b18a 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py @@ -7,12 +7,13 @@ from typing import Any, Dict import jsonschema -from airbyte_protocol.models import ConnectorSpecification -from connector_acceptance_test.utils import SecretDict from deepdiff import DeepDiff from hypothesis import HealthCheck, Verbosity, given, settings from hypothesis_jsonschema import from_schema +from airbyte_protocol.models import ConnectorSpecification +from connector_acceptance_test.utils import SecretDict + class BackwardIncompatibilityContext(Enum): SPEC = 1 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py index 06276ef4f32d1..a6583001a4968 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py @@ -10,8 +10,10 @@ from typing import List import dagger + from connector_acceptance_test.utils import SecretDict + IN_CONTAINER_CONNECTOR_PATH = Path("/connector") IN_CONTAINER_CONFIG_PATH = Path("/tmp/config.json") IN_CONTAINER_OUTPUT_PATH = Path("/tmp/output.txt") @@ -55,7 +57,7 @@ async def _run_with_config(container: dagger.Container, command: List[str], conf async def _run(container: dagger.Container, command: List[str]) -> dagger.Container: - return await (container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command)) + return await container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command) async def get_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py index 929309ca0e677..d704b21cf4dbc 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py @@ -11,6 +11,7 @@ import pytest from yaml import load + try: from yaml import CLoader as Loader except ImportError: diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py index 6ae6940f2d1d7..74e7a667eac40 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/compare.py @@ -11,6 +11,7 @@ import py from pprintpp import pformat + MAX_COLS = py.io.TerminalWriter().fullwidth MARGIN_LEFT = 20 GUTTER = 3 diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index 79165b95bf959..6d5ad2994c96d 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -14,11 +14,12 @@ import dagger import docker import pytest +from anyio import Path as AnyioPath +from pydantic import ValidationError + from airbyte_protocol.models import AirbyteMessage, ConfiguredAirbyteCatalog, OrchestratorType from airbyte_protocol.models import Type as AirbyteMessageType -from anyio import Path as AnyioPath from connector_acceptance_test.utils import SecretDict -from pydantic import ValidationError def splitlines_generator(input_string: str): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py index 06cd950636fe5..c366b6bc7d434 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/manifest_helper.py @@ -4,6 +4,7 @@ from airbyte_protocol.models import ConnectorSpecification + MANIFEST_FILE_NAMES = [ "manifest.yaml", "manifest.yml", diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py index fa92179b9385e..ac1bc6a68bd64 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py @@ -3,6 +3,8 @@ # import pytest +from connector_acceptance_test.utils.asserts import verify_records_schema + from airbyte_protocol.models import ( AirbyteRecordMessage, AirbyteStream, @@ -11,7 +13,6 @@ DestinationSyncMode, SyncMode, ) -from connector_acceptance_test.utils.asserts import verify_records_schema @pytest.fixture(name="record_schema") diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py index 3119a8d43511e..ba26286a1cef9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py @@ -6,11 +6,12 @@ from typing import MutableMapping, Union import pytest -from airbyte_protocol.models import AirbyteStream, ConnectorSpecification from connector_acceptance_test.tests.test_core import TestDiscovery as _TestDiscovery from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec from connector_acceptance_test.utils.backward_compatibility import NonBackwardCompatibleError, validate_previous_configs +from airbyte_protocol.models import AirbyteStream, ConnectorSpecification + from .conftest import does_not_raise diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py index 7435915090c75..7fd3d258c88ef 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -3,10 +3,12 @@ # import pytest -from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type from connector_acceptance_test.config import NoPrimaryKeyConfiguration from connector_acceptance_test.tests import test_core +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type + + pytestmark = pytest.mark.anyio @@ -14,7 +16,9 @@ "stream_configs, excluded_streams, expected_error_streams", [ pytest.param([{"name": "stream_with_primary_key", "primary_key": [["id"]]}], [], None, id="test_stream_with_primary_key_succeeds"), - pytest.param([{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails"), + pytest.param( + [{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails" + ), pytest.param([{"name": "report_stream"}], ["report_stream"], None, id="test_primary_key_excluded_from_test"), pytest.param( [ @@ -22,40 +26,55 @@ {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], [], ["himmel", "heiter"], id="test_multiple_streams_that_are_missing_primary_key"), + ], + [], + ["himmel", "heiter"], + id="test_multiple_streams_that_are_missing_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["himmel", "heiter"], None, id="test_multiple_streams_that_exclude_primary_key"), + ], + ["himmel", "heiter"], + None, + id="test_multiple_streams_that_exclude_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["heiter"], ["himmel"], id="test_multiple_streams_missing_primary_key_or_excluded"), + ], + ["heiter"], + ["himmel"], + id="test_multiple_streams_missing_primary_key_or_excluded", + ), ], ) async def test_streams_define_primary_key(mocker, stream_configs, excluded_streams, expected_error_streams): t = test_core.TestConnectorAttributes() - streams = [AirbyteStream.parse_obj({ - "name": stream_config.get("name"), - "json_schema": {}, - "default_cursor_field": ["updated_at"], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": stream_config.get("primary_key"), - }) for stream_config in stream_configs] + streams = [ + AirbyteStream.parse_obj( + { + "name": stream_config.get("name"), + "json_schema": {}, + "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": stream_config.get("primary_key"), + } + ) + for stream_config in stream_configs + ] streams_without_primary_key = [NoPrimaryKeyConfiguration(name=stream, bypass_reason="") for stream in excluded_streams] docker_runner_mock = mocker.MagicMock( - call_discover=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))] - ) + call_discover=mocker.AsyncMock(return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))]) ) if expected_error_streams: @@ -64,7 +83,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) streams_in_error_message = [stream_name for stream_name in expected_error_streams if stream_name in e.value.args[0]] assert streams_in_error_message == expected_error_streams @@ -73,7 +92,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) @@ -106,26 +125,22 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea "Has `allowdHosts` but no `hosts`", "Has `hosts` but it's empty list", "Has non-empty `hosts`", - ] + ], ) async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) - + @pytest.mark.parametrize( "metadata_yaml, should_raise_assert_error, expected_error", @@ -156,22 +171,18 @@ async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise "Has `suggestedStreams` but no `streams`", "Has `streams` but it's empty list", "Has non-empty `streams`", - ] + ], ) async def test_certified_connector_has_suggested_streams(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml - ) \ No newline at end of file + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml + ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py index c7399c1fbe737..746d6b1166a4b 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py @@ -8,6 +8,8 @@ from pathlib import Path import pytest +from connector_acceptance_test.utils import connector_runner + from airbyte_protocol.models import ( AirbyteControlConnectorConfigMessage, AirbyteControlMessage, @@ -16,7 +18,7 @@ OrchestratorType, ) from airbyte_protocol.models import Type as AirbyteMessageType -from connector_acceptance_test.utils import connector_runner + pytestmark = pytest.mark.anyio @@ -121,7 +123,6 @@ def test_persist_new_configuration( async def test_get_connector_container(mocker): - dagger_client = mocker.AsyncMock() os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"] = "test_tarball_path" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index 71799cfead083..d4dcec8c98552 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -6,6 +6,18 @@ from unittest.mock import MagicMock, patch import pytest +from connector_acceptance_test.config import ( + BasicReadTestConfig, + Config, + DiscoveryTestConfig, + ExpectedRecordsConfig, + FileTypesConfig, + IgnoredFieldsConfiguration, + UnsupportedFileTypeConfig, +) +from connector_acceptance_test.tests import test_core +from jsonschema.exceptions import SchemaError + from airbyte_protocol.models import ( AirbyteErrorTraceMessage, AirbyteLogMessage, @@ -28,20 +40,10 @@ TraceType, Type, ) -from connector_acceptance_test.config import ( - BasicReadTestConfig, - Config, - DiscoveryTestConfig, - ExpectedRecordsConfig, - FileTypesConfig, - IgnoredFieldsConfiguration, - UnsupportedFileTypeConfig, -) -from connector_acceptance_test.tests import test_core -from jsonschema.exceptions import SchemaError from .conftest import does_not_raise + pytestmark = pytest.mark.anyio @@ -100,18 +102,11 @@ def test_discovery_uniquely_named_streams(): "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": ["null", "integer"] - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": ["null", "integer"]}}, + }, }, - True + True, ), ( { @@ -119,38 +114,22 @@ def test_discovery_uniquely_named_streams(): "type": ["null", "object"], "properties": { "amount": "integer", - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - True + True, ), ( { "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - False - ) + False, + ), ], ) def test_streams_have_valid_json_schemas(schema, should_fail): @@ -639,9 +618,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -658,9 +635,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -673,9 +648,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -688,9 +661,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -707,9 +678,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -722,9 +691,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -883,7 +850,7 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca _DEFAULT_RECORD_CONFIG, [ {"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}, - {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]} + {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]}, ], {"test_stream": [{"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}]}, None, @@ -911,13 +878,13 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca ), # Expected is in actual but not in order (for case when exact_order=True) ( - {"type": "object"}, - {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, - ExpectedRecordsConfig(exact_order=True, path="foobar"), - [{"constant_field": "not in order"}, {"constant_field": "must equal"}], - {"test_stream": [{"constant_field": "must equal"}]}, - None, - does_not_raise(), + {"type": "object"}, + {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, + ExpectedRecordsConfig(exact_order=True, path="foobar"), + [{"constant_field": "not in order"}, {"constant_field": "must equal"}], + {"test_stream": [{"constant_field": "must equal"}]}, + None, + does_not_raise(), ), # Match by primary key ( @@ -985,12 +952,14 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({ - "name": "test_stream", - "json_schema": schema, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": primary_key - }), + stream=AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": schema, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": primary_key, + } + ), sync_mode="full_refresh", destination_sync_mode="overwrite", ) @@ -998,7 +967,10 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor ) docker_runner_mock = mocker.MagicMock( call_read=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) for record in records] + return_value=[ + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) + for record in records + ] ) ) t = test_core.TestBasicRead() @@ -1954,8 +1926,8 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p ] ) stream = AirbyteStreamState( - stream_descriptor=StreamDescriptor(name='test_stream_0', namespace=None), - stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True) + stream_descriptor=StreamDescriptor(name="test_stream_0", namespace=None), + stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True), ) async_stream_output = [ AirbyteMessage( @@ -1989,7 +1961,7 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.COMPLETE ), ), - ) + ), ] if not state_message_params: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py index 5a602e85a2e75..800860214f629 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py @@ -3,27 +3,28 @@ from pathlib import Path import pytest -from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test import conftest from connector_acceptance_test.tests.test_core import TestConnectorDocumentation as _TestConnectorDocumentation +from airbyte_protocol.models import ConnectorSpecification + @pytest.mark.parametrize( "connector_spec, docs_path, should_fail", ( - # SUCCESS: required field from spec exists in Prerequisites section - ( - {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, - "data/docs/incorrect_not_all_structure.md", - False - ), - # FAIL: required field from spec does not exist in Prerequisites section - ( - {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, - "data/docs/incorrect_not_all_structure.md", - True - ) - ) + # SUCCESS: required field from spec exists in Prerequisites section + ( + {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, + "data/docs/incorrect_not_all_structure.md", + False, + ), + # FAIL: required field from spec does not exist in Prerequisites section + ( + {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + ), ) def test_documentation_prerequisites_section(connector_spec, docs_path, should_fail): t = _TestConnectorDocumentation() @@ -41,35 +42,35 @@ def test_documentation_prerequisites_section(connector_spec, docs_path, should_f @pytest.mark.parametrize( "metadata, docs_path, should_fail, failure", ( - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - "Missing headers:", - ), - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "Oracle Netsuite"}}, - "data/docs/with_not_required_steps.md", - True, - "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", - ), - # # SUCCESS: Docs follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - "", - ), - # Fail: Incorrect header order - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_header_order.md", - True, - "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", - ), - ) + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + "Missing headers:", + ), + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "Oracle Netsuite"}}, + "data/docs/with_not_required_steps.md", + True, + "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", + ), + # # SUCCESS: Docs follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + "", + ), + # Fail: Incorrect header order + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_header_order.md", + True, + "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", + ), + ), ) def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, failure): t = _TestConnectorDocumentation() @@ -89,25 +90,25 @@ def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, fai @pytest.mark.parametrize( "metadata, docs_path, should_fail", ( - # FAIL: Prerequisites section does not follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct_all_description_exist.md", - False, - ), - ) + # FAIL: Prerequisites section does not follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct_all_description_exist.md", + False, + ), + ), ) def test_docs_description(mocker, metadata, docs_path, should_fail): mocker.patch.object(conftest.pytest, "fail") @@ -140,7 +141,7 @@ def test_docs_description(mocker, metadata, docs_path, should_fail): "data/docs/correct.md", False, ), - ) + ), ) def test_docs_urls(docs_path, should_fail): t = _TestConnectorDocumentation() diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py index c6c617be86934..bc76474d1c75d 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py @@ -6,7 +6,6 @@ import time import pytest -from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test import conftest from connector_acceptance_test.config import ( BasicReadTestConfig, @@ -16,6 +15,8 @@ IgnoredFieldsConfiguration, ) +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.mark.parametrize( "test_strictness_level, basic_read_test_config, expect_test_failure", diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index 05724361dab07..7b9307f26fc37 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -11,6 +11,17 @@ from unittest.mock import MagicMock, patch import pytest +from connector_acceptance_test.config import ( + Config, + EmptyStreamConfiguration, + FutureStateConfig, + FutureStateCursorFormatConfiguration, + IncrementalConfig, +) +from connector_acceptance_test.tests import test_incremental +from connector_acceptance_test.tests.test_incremental import TestIncremental as _TestIncremental +from connector_acceptance_test.tests.test_incremental import future_state_configuration_fixture, future_state_fixture + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,16 +38,7 @@ SyncMode, Type, ) -from connector_acceptance_test.config import ( - Config, - EmptyStreamConfiguration, - FutureStateConfig, - FutureStateCursorFormatConfiguration, - IncrementalConfig, -) -from connector_acceptance_test.tests import test_incremental -from connector_acceptance_test.tests.test_incremental import TestIncremental as _TestIncremental -from connector_acceptance_test.tests.test_incremental import future_state_configuration_fixture, future_state_fixture + pytestmark = [ pytest.mark.anyio, @@ -56,7 +58,10 @@ def build_state_message(state: dict) -> AirbyteMessage: def build_per_stream_state_message( - descriptor: StreamDescriptor, stream_state: Optional[dict[str, Any]], data: Optional[dict[str, Any]] = None, source_stats: Optional[dict[str, Any]] = None + descriptor: StreamDescriptor, + stream_state: Optional[dict[str, Any]], + data: Optional[dict[str, Any]] = None, + source_stats: Optional[dict[str, Any]] = None, ) -> AirbyteMessage: if data is None: data = stream_state @@ -70,7 +75,7 @@ def build_per_stream_state_message( type=AirbyteStateType.STREAM, stream=AirbyteStreamState(stream_descriptor=descriptor, stream_state=stream_state_blob), sourceStats=AirbyteStateStats(**source_stats), - data=data + data=data, ), ) @@ -232,20 +237,40 @@ async def test_incremental_two_sequential_reads( [ {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, ], # Read after 2022-05-10. This is the second (last) subsequent read. [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, - ] + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, + ], ], IncrementalConfig(), does_not_raise(), @@ -258,9 +283,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 0.0}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_records_on_first_read_skips_stream", @@ -274,9 +297,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_states_on_first_read_skips_stream", @@ -293,13 +314,28 @@ async def test_incremental_two_sequential_reads( ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-08"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], ], IncrementalConfig(), @@ -373,7 +409,10 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse call_read_output_messages = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), ) if record["type"] == Type.STATE else build_record_message(record["name"], record["data"]) @@ -382,7 +421,10 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse call_read_with_state_output_messages = [ [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), ) if record["type"] == Type.STATE else build_record_message(stream=record["name"], data=record["data"]) @@ -416,22 +458,20 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse [ {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, ], [], [], - id="combine_three_duplicates_into_a_single_state_message" + id="combine_three_duplicates_into_a_single_state_message", ), pytest.param( [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}} - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, ], + [{"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}], [0.0], - id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal" + id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal", ), pytest.param( [ @@ -443,27 +483,33 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], - [10.0, 3.0, 0.0] - ) + [10.0, 3.0, 0.0], + ), ], ) async def test_get_unique_state_messages(non_unique_states, expected_unique_states, expected_record_count_per_state): non_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in non_unique_states ] expected_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in expected_unique_states ] @@ -471,7 +517,9 @@ async def test_get_unique_state_messages(non_unique_states, expected_unique_stat assert len(actual_unique_states) == len(expected_unique_states) if len(expected_unique_states): - for actual_state_data, expected_state, expected_record_count in zip(actual_unique_states, expected_unique_states, expected_record_count_per_state): + for actual_state_data, expected_state, expected_record_count in zip( + actual_unique_states, expected_unique_states, expected_record_count_per_state + ): actual_state, actual_record_count = actual_state_data assert actual_state == expected_state assert actual_record_count == expected_record_count @@ -517,7 +565,8 @@ async def test_config_skip_test(mocker): IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration())), [], {"type": "object", "properties": {"date": {"type": "str"}}}, - pytest.raises(AssertionError), id="Error because incremental stream should always emit state messages" + pytest.raises(AssertionError), + id="Error because incremental stream should always emit state messages", ), pytest.param( [ @@ -561,20 +610,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -594,25 +632,16 @@ async def test_config_skip_test(mocker): ), ) ], - IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$"))), + IncrementalConfig( + future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$")) + ), [ { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -637,20 +666,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-05-08T03:04:45.139-0700" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-05-08T03:04:45.139-0700"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -676,20 +694,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": 10000000.0 - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": 10000000.0}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": ["int", "null"]}}}, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py index 89536ae245421..8cfcd6d21bb77 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py @@ -7,6 +7,9 @@ import pendulum import pytest +from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure +from pydantic import BaseModel + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,6 @@ SyncMode, Type, ) -from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure -from pydantic import BaseModel def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py index 188b8e39bc181..10659cc83258f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_plugin.py @@ -5,6 +5,7 @@ import pytest from connector_acceptance_test import config, plugin + HIGH_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.high LOW_TEST_STRICTNESS_LEVEL = config.Config.TestStrictnessLevel.low diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 4b4b1d456e756..50ea4c0efa258 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -5,10 +5,11 @@ from typing import Any, Callable, Dict import pytest -from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test import conftest from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec +from airbyte_protocol.models import ConnectorSpecification + from .conftest import does_not_raise @@ -681,7 +682,7 @@ def test_enum_usage(connector_spec, should_fail): }, ), "", - ) + ), ], ) def test_validate_oauth_flow(connector_spec, expected_error): @@ -698,227 +699,153 @@ def test_validate_oauth_flow(connector_spec, expected_error): [ # FAIL: OAuth is not default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, }, - ] - } - } + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "Oauth method should be a default option. Current default method is access_token." + }, + }, + }, + ), + "Oauth method should be a default option. Current default method is access_token.", ), # SUCCESS: Oauth is default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - } - ] - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { "type": "string", "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "" + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "", ), # SUCCESS: no advancedAuth specified (ConnectorSpecification(connectionSpecification={}), ""), @@ -992,60 +919,60 @@ def test_validate_oauth_flow(connector_spec, expected_error): ), # SUCCESS: Skipped: no predicate key. ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": {"type": "object"}, - "credentials": { - "type": "object", - "properties": { - "auth_type": {"type": "string", "const": "oauth2.0"}, - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - "token_expiry_date": {"type": "string", "format": "date-time"}, - }, + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, }, }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, - "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"], - }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, - "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, - }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, }, }, - ), - "Advanced Auth object does not have predicate_key, only one option to authenticate.", - ) - ] + }, + ), + "Advanced Auth object does not have predicate_key, only one option to authenticate.", + ), + ], ) def test_validate_auth_default_method(connector_spec, expected_error): t = _TestSpec() @@ -1106,69 +1033,66 @@ def test_additional_properties_is_true(connector_spec, expectation): ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string"}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": {"client_secret": {"type": "string"}, "access_token": {"type": "string", "airbyte_secret": True}}, + } + }, }, - True + True, ), ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string", "airbyte_secret": True}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": { + "client_secret": {"type": "string", "airbyte_secret": True}, + "access_token": {"type": "string", "airbyte_secret": True}, + }, + } + }, }, - False + False, ), ( - { - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - ] - } - } + { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + ], + } }, - True + }, + True, ), ({"type": "object", "properties": {"auth": {"oneOf": [{"api_token": {"type": "string"}}]}}}, True), ( @@ -1187,7 +1111,9 @@ def test_airbyte_secret(mocker, connector_spec, should_fail): t = _TestSpec() logger = mocker.Mock() t.test_secret_is_properly_marked( - {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret") + {"connectionSpecification": connector_spec}, + logger, + ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret"), ) if should_fail: conftest.pytest.fail.assert_called_once() diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py index d767ffa3a0f6d..bf8ca2d41c122 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py @@ -7,6 +7,9 @@ import pytest from _pytest.outcomes import Failed +from connector_acceptance_test.config import ConnectionTestConfig, IgnoredFieldsConfiguration +from connector_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh + from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,7 @@ SyncMode, Type, ) -from connector_acceptance_test.config import ConnectionTestConfig, IgnoredFieldsConfiguration -from connector_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh + pytestmark = pytest.mark.anyio diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py index 4316d871ca715..d9e1628a29a02 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py @@ -14,11 +14,12 @@ import pytest import yaml -from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test.config import EmptyStreamConfiguration from connector_acceptance_test.utils import common from connector_acceptance_test.utils.compare import make_hashable +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + def not_sorted_data(): return { diff --git a/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py index 9c063d1a2226b..1ce3a6008fc0e 100644 --- a/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py +++ b/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py index de6cab2f4cbdd..43a585a395d4f 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py @@ -9,9 +9,10 @@ from uuid import uuid4 import boto3 +from botocore.exceptions import ClientError + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type -from botocore.exceptions import ClientError class DestinationAmazonSqs(Destination): @@ -80,7 +81,6 @@ def set_message_fifo_properties(self, message, message_group_id, use_content_ded def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - # Required propeties queue_url = config["queue_url"] queue_region = config["region"] diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py index 1517a69e1bcc6..4af175c0cc5d1 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/integration_tests/integration_test.py @@ -7,9 +7,10 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, Status, SyncMode from destination_amazon_sqs import DestinationAmazonSqs +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, Status, SyncMode + @pytest.fixture(name="config") def config_fixture() -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/main.py b/airbyte-integrations/connectors/destination-amazon-sqs/main.py index bc6076972a29a..2761b5787225a 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/main.py @@ -7,5 +7,6 @@ from destination_amazon_sqs import DestinationAmazonSqs + if __name__ == "__main__": DestinationAmazonSqs().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py index 9ebad9efae7d9..e0d84ddc072d1 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/unit_tests/unit_test.py @@ -8,13 +8,14 @@ from typing import Any, Mapping import boto3 -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status from destination_amazon_sqs import DestinationAmazonSqs # from airbyte_cdk.sources.source import Source from moto import mock_iam, mock_sqs from moto.core import set_initial_no_auth_action_count +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status + @mock_iam def create_user_with_all_permissions(): diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py index 01d805ecd782c..6e36fa02ba5e9 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/config.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/config.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class AstraIndexingModel(BaseModel): astra_db_app_token: str = Field( diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py index 1c79c9aa1d485..20a365c8c880c 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/destination.py @@ -16,6 +16,7 @@ from destination_astra.config import ConfigModel from destination_astra.indexer import AstraIndexer + BATCH_SIZE = 100 diff --git a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py index ee936ae1f5b44..8c64860c8c5f8 100644 --- a/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py +++ b/airbyte-integrations/connectors/destination-astra/destination_astra/indexer.py @@ -6,6 +6,7 @@ from typing import Optional import urllib3 + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception @@ -13,6 +14,7 @@ from destination_astra.astra_client import AstraClient from destination_astra.config import AstraIndexingModel + # do not flood the server with too many connections in parallel PARALLELISM_LIMIT = 20 diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py index b9d1aac8ae3c9..2ab044fbdf47d 100644 --- a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -4,27 +4,26 @@ import logging -from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_astra.astra_client import AstraClient from destination_astra.config import ConfigModel from destination_astra.destination import DestinationAstra +from airbyte_cdk.destinations.vector_db_based.embedder import create_from_config +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status -class AstraIntegrationTest(BaseIntegrationTest): +class AstraIntegrationTest(BaseIntegrationTest): def test_check_valid_config(self): outcome = DestinationAstra().check(logging.getLogger("airbyte"), self.config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): - invalid_config = self.config + invalid_config = self.config invalid_config["embedding"]["openai_key"] = 123 - outcome = DestinationAstra().check( - logging.getLogger("airbyte"), invalid_config) + outcome = DestinationAstra().check(logging.getLogger("airbyte"), invalid_config) assert outcome.status == Status.FAILED def test_write(self): @@ -32,11 +31,7 @@ def test_write(self): embedder = create_from_config(db_config.embedding, db_config.processing) db_creds = db_config.indexing astra_client = AstraClient( - db_creds.astra_db_endpoint, - db_creds.astra_db_app_token, - db_creds.astra_db_keyspace, - embedder.embedding_dimensions, - "cosine" + db_creds.astra_db_endpoint, db_creds.astra_db_app_token, db_creds.astra_db_keyspace, embedder.embedding_dimensions, "cosine" ) astra_client.delete_documents(collection_name=db_creds.collection, filter={}) @@ -49,4 +44,3 @@ def test_write(self): outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) assert astra_client.count_documents(db_creds.collection) == 2 - diff --git a/airbyte-integrations/connectors/destination-astra/main.py b/airbyte-integrations/connectors/destination-astra/main.py index 53b96b2b39ecb..f56f6b8ac36ad 100644 --- a/airbyte-integrations/connectors/destination-astra/main.py +++ b/airbyte-integrations/connectors/destination-astra/main.py @@ -7,5 +7,6 @@ from destination_astra import DestinationAstra + if __name__ == "__main__": DestinationAstra().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py index a4094d41f6956..b25fe4966b320 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_astra.config import ConfigModel from destination_astra.destination import DestinationAstra +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationAstra(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py index ebb1d41e230a0..312956df50060 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -5,10 +5,11 @@ import pytest import urllib3 -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_astra.config import AstraIndexingModel from destination_astra.indexer import AstraIndexer +from airbyte_cdk.models import ConfiguredAirbyteCatalog + def create_astra_indexer(): config = AstraIndexingModel( @@ -142,7 +143,9 @@ def test_astra_pre_sync(): ("other_collection", None, 3, False, "mycollection collection does not exist."), ( ["mycollection"], - urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.")), + urllib3.exceptions.MaxRetryError( + None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.") + ), 3, False, "Failed to resolve environment", @@ -157,12 +160,16 @@ def test_astra_check(collection_name, describe_throws, reported_dimensions, chec indexer.client.create_collection = MagicMock() indexer.client.find_collections = MagicMock() - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] if describe_throws: indexer.client.find_collections.side_effect = describe_throws else: - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] result = indexer.check() if check_succeeds: diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py index 0c72637d5c848..22de92c7373c7 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py @@ -10,15 +10,17 @@ import boto3 import botocore import pandas as pd -from airbyte_cdk.destinations import Destination from awswrangler import _data_types from botocore.credentials import AssumeRoleCredentialFetcher, CredentialResolver, DeferredRefreshableCredentials, JSONFileCache from botocore.exceptions import ClientError from retrying import retry +from airbyte_cdk.destinations import Destination + from .config_reader import CompressionCodec, ConnectorConfig, CredentialsType, OutputFormat from .constants import BOOLEAN_VALUES, EMPTY_VALUES + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py index 223312f5d05ba..72cd4acebb2d4 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py @@ -8,14 +8,16 @@ from typing import Any, Dict, Iterable, Mapping import pandas as pd +from botocore.exceptions import ClientError, InvalidRegionError + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, AirbyteStateType, ConfiguredAirbyteCatalog, Status, Type -from botocore.exceptions import ClientError, InvalidRegionError from .aws import AwsHandler from .config_reader import ConnectorConfig from .stream_writer import StreamWriter + logger = logging.getLogger("airbyte") # Flush records every 25000 records to limit memory consumption @@ -34,7 +36,6 @@ def _get_random_string(length: int) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py index d7ee96e276882..ba1972b24a83e 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py @@ -9,12 +9,14 @@ from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd + from airbyte_cdk.models import ConfiguredAirbyteStream, DestinationSyncMode from .aws import AwsHandler from .config_reader import ConnectorConfig, PartitionOptions from .constants import EMPTY_VALUES, GLUE_TYPE_MAPPING_DECIMAL, GLUE_TYPE_MAPPING_DOUBLE, PANDAS_TYPE_MAPPING + # By default we set glue decimal type to decimal(28,25) # this setting matches that precision. getcontext().prec = 25 diff --git a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py index 29f792fbc02e8..4b871b2dc6679 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py @@ -9,6 +9,10 @@ import awswrangler as wr import pytest +from destination_aws_datalake import DestinationAwsDatalake +from destination_aws_datalake.aws import AwsHandler +from destination_aws_datalake.config_reader import ConnectorConfig + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -24,9 +28,7 @@ SyncMode, Type, ) -from destination_aws_datalake import DestinationAwsDatalake -from destination_aws_datalake.aws import AwsHandler -from destination_aws_datalake.config_reader import ConnectorConfig + logger = logging.getLogger("airbyte") @@ -95,13 +97,13 @@ def test_check_invalid_aws_account_config(invalid_account_config: Mapping): def _state(stream: str, data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage( - type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_state=data, - stream_descriptor=StreamDescriptor(name=stream, namespace=None) - ) - )) + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState(stream_state=data, stream_descriptor=StreamDescriptor(name=stream, namespace=None)), + ), + ) def _record(stream: str, str_value: str, int_value: int, date_value: datetime) -> AirbyteMessage: diff --git a/airbyte-integrations/connectors/destination-aws-datalake/main.py b/airbyte-integrations/connectors/destination-aws-datalake/main.py index 23a5962dcc72e..0aca839fd2148 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/main.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/main.py @@ -7,5 +7,6 @@ from destination_aws_datalake import DestinationAwsDatalake + if __name__ == "__main__": DestinationAwsDatalake().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py index e981907cec407..d27168e067cad 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py @@ -9,12 +9,13 @@ import numpy as np import pandas as pd -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from destination_aws_datalake import DestinationAwsDatalake from destination_aws_datalake.aws import AwsHandler from destination_aws_datalake.config_reader import ConnectorConfig from destination_aws_datalake.stream_writer import DictEncoder, StreamWriter +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + def get_config() -> Mapping[str, Any]: with open("unit_tests/fixtures/config.json", "r") as f: diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py index 557d4f527986a..3d41cfd6a6a19 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py @@ -4,6 +4,8 @@ from typing import Literal, Optional, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import ( AzureOpenAIEmbeddingConfigModel, CohereEmbeddingConfigModel, @@ -13,7 +15,6 @@ OpenAIEmbeddingConfigModel, VectorDBConfigModel, ) -from pydantic import BaseModel, Field class HttpMode(BaseModel): @@ -41,7 +42,6 @@ class Config: class ChromaIndexingConfigModel(BaseModel): - auth_method: Union[PersistentMode, HttpMode] = Field( ..., title="Connection Mode", description="Mode how to connect to Chroma", discriminator="mode", type="object", order=0 ) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py index b0926ffd3c659..7d04bfaa045d0 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -23,11 +23,11 @@ from destination_chroma.indexer import ChromaIndexer from destination_chroma.no_embedder import NoEmbedder + BATCH_SIZE = 128 class DestinationChroma(Destination): - indexer: Indexer embedder: Embedder @@ -42,7 +42,6 @@ def _init_indexer(self, config: ConfigModel): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py index 3b0d741d7feb6..10ed56ec44b91 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py @@ -6,12 +6,13 @@ import uuid import chromadb +from chromadb.config import Settings + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode -from chromadb.config import Settings from destination_chroma.config import ChromaIndexingConfigModel from destination_chroma.utils import is_valid_collection_name diff --git a/airbyte-integrations/connectors/destination-chroma/main.py b/airbyte-integrations/connectors/destination-chroma/main.py index 88af98a9500c4..146cc0699350b 100644 --- a/airbyte-integrations/connectors/destination-chroma/main.py +++ b/airbyte-integrations/connectors/destination-chroma/main.py @@ -7,5 +7,6 @@ from destination_chroma import DestinationChroma + if __name__ == "__main__": DestinationChroma().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py index 52460e5331721..b7c3256fc450d 100644 --- a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_chroma.config import ConfigModel from destination_chroma.destination import DestinationChroma +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationChroma(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py index f1a9bf493d57e..88a1160496dc9 100644 --- a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py @@ -5,10 +5,11 @@ import unittest from unittest.mock import Mock -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode from destination_chroma.config import ChromaIndexingConfigModel from destination_chroma.indexer import ChromaIndexer +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode + class TestChromaIndexer(unittest.TestCase): def setUp(self): @@ -30,7 +31,6 @@ def setUp(self): self.mock_client.get_collection = Mock() def test_valid_collection_name(self): - test_configs = [ ({"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, None), ( diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/client.py b/airbyte-integrations/connectors/destination-convex/destination_convex/client.py index 3cace64444da2..00aba7fbcff26 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/client.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/client.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping import requests + from destination_convex.config import ConvexConfig diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/config.py b/airbyte-integrations/connectors/destination-convex/destination_convex/config.py index 1c5dc8ca2bccc..3eba952a5e360 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/config.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/config.py @@ -4,6 +4,7 @@ from typing import TypedDict + ConvexConfig = TypedDict( "ConvexConfig", { diff --git a/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py b/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py index 4a139ceae895c..284916c8fb370 100644 --- a/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py +++ b/airbyte-integrations/connectors/destination-convex/destination_convex/destination.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, Optional, cast import requests + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import ( AirbyteConnectionStatus, diff --git a/airbyte-integrations/connectors/destination-convex/main.py b/airbyte-integrations/connectors/destination-convex/main.py index 7a572b8ee791c..2426c8f42fa3a 100644 --- a/airbyte-integrations/connectors/destination-convex/main.py +++ b/airbyte-integrations/connectors/destination-convex/main.py @@ -7,5 +7,6 @@ from destination_convex import DestinationConvex + if __name__ == "__main__": DestinationConvex().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py index d39027b38692f..534dafa728593 100644 --- a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py @@ -7,6 +7,10 @@ import pytest import responses +from destination_convex.client import ConvexClient +from destination_convex.config import ConvexConfig +from destination_convex.destination import DestinationConvex + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -18,9 +22,7 @@ SyncMode, Type, ) -from destination_convex.client import ConvexClient -from destination_convex.config import ConvexConfig -from destination_convex.destination import DestinationConvex + DEDUP_TABLE_NAME = "dedup_stream" DEDUP_INDEX_FIELD = "int_col" diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py index 10728e374f54d..faa12d0c89575 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/client.py @@ -9,6 +9,7 @@ from cumulio.cumulio import Cumulio # type: ignore + # def _retry_with_backoff( # fn: Callable, # backoff_times_in_seconds: list[int] diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py index 61c6c5ac4afb0..68ea73a6fdf0b 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py @@ -11,6 +11,7 @@ from destination_cumulio.client import CumulioClient from destination_cumulio.writer import CumulioWriter + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py index 545241d463e7d..1ecf57279111a 100644 --- a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py @@ -8,6 +8,9 @@ from typing import Any, Dict, Mapping import pytest +from destination_cumulio import DestinationCumulio +from destination_cumulio.client import CumulioClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -20,8 +23,6 @@ SyncMode, Type, ) -from destination_cumulio import DestinationCumulio -from destination_cumulio.client import CumulioClient @pytest.fixture(name="logger") diff --git a/airbyte-integrations/connectors/destination-cumulio/main.py b/airbyte-integrations/connectors/destination-cumulio/main.py index 3ad0d71122069..ef230369f7933 100644 --- a/airbyte-integrations/connectors/destination-cumulio/main.py +++ b/airbyte-integrations/connectors/destination-cumulio/main.py @@ -7,5 +7,6 @@ from destination_cumulio import DestinationCumulio + if __name__ == "__main__": DestinationCumulio().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py index 258e8ff2a578a..48f538b20d676 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py @@ -9,6 +9,7 @@ import pytest from destination_cumulio.client import CumulioClient + # "# type: ignore" was added in several places to avoid mypy complaining about patching functions with MagicMock diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py index 4805fb51ecf5c..2c278430dd6ca 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py @@ -9,6 +9,8 @@ from unittest.mock import MagicMock, call, patch import pytest +from destination_cumulio.destination import DestinationCumulio + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -20,7 +22,6 @@ SyncMode, Type, ) -from destination_cumulio.destination import DestinationCumulio @pytest.fixture(name="logger") diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py index ac921c7ef5c41..4c95b3e670e06 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py @@ -8,9 +8,10 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from destination_cumulio.writer import CumulioWriter +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.fixture(name="logger") def logger_fixture() -> MagicMock: diff --git a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py index f8da4b0984bde..01a345b41249f 100644 --- a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py +++ b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py @@ -16,6 +16,7 @@ from .writer import create_databend_wirter + logger = getLogger("airbyte") @@ -23,7 +24,6 @@ class DestinationDatabend(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ TODO Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py index a40494c4e048b..9bca56ba42247 100644 --- a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from typing import Any, Dict, List, Mapping import pytest +from destination_databend import DestinationDatabend +from destination_databend.client import DatabendClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_databend import DestinationDatabend -from destination_databend.client import DatabendClient @pytest.fixture(name="databendConfig") diff --git a/airbyte-integrations/connectors/destination-databend/main.py b/airbyte-integrations/connectors/destination-databend/main.py index 7482c00577de5..67d4d20d9cad1 100644 --- a/airbyte-integrations/connectors/destination-databend/main.py +++ b/airbyte-integrations/connectors/destination-databend/main.py @@ -7,5 +7,6 @@ from destination_databend import DestinationDatabend + if __name__ == "__main__": DestinationDatabend().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py index 2372f49736fbb..8047bef61ebf5 100644 --- a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py +++ b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py @@ -6,6 +6,9 @@ from typing import Dict from unittest.mock import AsyncMock, MagicMock, call, patch +from destination_databend.destination import DatabendClient, DestinationDatabend +from pytest import fixture + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -16,8 +19,6 @@ SyncMode, Type, ) -from destination_databend.destination import DatabendClient, DestinationDatabend -from pytest import fixture @fixture @@ -117,14 +118,14 @@ def test_connection(config: Dict[str, str], logger: MagicMock) -> None: @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_append( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ) -> None: catalog = ConfiguredAirbyteCatalog(streams=[configured_stream1, configured_stream2]) @@ -141,14 +142,14 @@ def test_sql_write_append( @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_overwrite( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ): # Overwrite triggers a delete configured_stream1.destination_sync_mode = DestinationSyncMode.overwrite diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index 8e74038d41e67..facbeb3344c9c 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -2,6 +2,7 @@ import datetime import json +import logging import os import re import uuid @@ -12,7 +13,6 @@ import duckdb import pyarrow as pa -import logging from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py index d985da9706db0..b3320efb79fee 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py @@ -15,6 +15,10 @@ import duckdb import pytest +from destination_duckdb import DestinationDuckdb +from destination_duckdb.destination import CONFIG_MOTHERDUCK_API_KEY +from faker import Faker + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,14 +31,10 @@ SyncMode, Type, ) -from destination_duckdb import DestinationDuckdb -from destination_duckdb.destination import CONFIG_MOTHERDUCK_API_KEY -from faker import Faker + CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +45,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -102,6 +100,7 @@ def test_large_table_name() -> str: rand_string = "".join(random.choice(letters) for _ in range(10)) return f"airbyte_integration_{rand_string}" + @pytest.fixture def table_schema() -> str: schema = {"type": "object", "properties": {"column1": {"type": ["null", "string"]}}} @@ -110,7 +109,9 @@ def table_schema() -> str: @pytest.fixture def configured_catalogue( - test_table_name: str, test_large_table_name: str, table_schema: str, + test_table_name: str, + test_large_table_name: str, + table_schema: str, ) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( stream=AirbyteStream( @@ -159,9 +160,7 @@ def airbyte_message2(test_table_name: str): @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.mark.disable_autouse @@ -208,9 +207,7 @@ def test_write( if motherduck_api_key: duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: cursor = con.execute( "SELECT _airbyte_ab_id, _airbyte_emitted_at, _airbyte_data " @@ -222,21 +219,20 @@ def test_write( assert result[0][2] == json.dumps(airbyte_message1.record.data) assert result[1][2] == json.dumps(airbyte_message2.record.data) + def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, - data={"key1": fake.first_name() , "key2": fake.ssn()}, + data={"key1": fake.first_name(), "key2": fake.ssn()}, emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -250,28 +246,28 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, # Throw in empty nested objects and see how pyarrow deals with them. - data={"key1": fake.first_name(), - "key2": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "nested1": {} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "dictionary1":{} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": "True" if random.random() < 0.5 else True - } - } - } - if random.random() < 0.9 else {}, - + data={ + "key1": fake.first_name(), + "key2": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "nested1": {} + if random.random() < 0.1 + else { + "key3": fake.first_name(), + "key4": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "dictionary1": {} + if random.random() < 0.1 + else {"key3": fake.first_name(), "key4": "True" if random.random() < 0.5 else True}, + }, + } + if random.random() < 0.9 + else {}, emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -281,10 +277,15 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab TOTAL_RECORDS = 5_000 BATCH_WRITE_SIZE = 1000 + @pytest.mark.slow -@pytest.mark.parametrize("airbyte_message_generator,explanation", - [(_airbyte_messages, "Test writing a large number of simple json objects."), - (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema.")] ) +@pytest.mark.parametrize( + "airbyte_message_generator,explanation", + [ + (_airbyte_messages, "Test writing a large number of simple json objects."), + (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema."), + ], +) def test_large_number_of_writes( config: Dict[str, str], request, @@ -309,13 +310,8 @@ def test_large_number_of_writes( duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: - cursor = con.execute( - "SELECT count(1) " - f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}" - ) + cursor = con.execute("SELECT count(1) " f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}") result = cursor.fetchall() assert result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-duckdb/main.py b/airbyte-integrations/connectors/destination-duckdb/main.py index 5aca3c1667452..1353a0f6298f6 100644 --- a/airbyte-integrations/connectors/destination-duckdb/main.py +++ b/airbyte-integrations/connectors/destination-duckdb/main.py @@ -5,5 +5,6 @@ from destination_duckdb.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py index 4f5aeefa09b7b..7a2616579d570 100644 --- a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py @@ -4,6 +4,7 @@ import os import tempfile from unittest.mock import Mock, patch + import pytest from destination_duckdb.destination import CONFIG_DEFAULT_SCHEMA, DestinationDuckdb, validated_sql_name @@ -15,6 +16,7 @@ def test_validated_sql_name() -> None: with pytest.raises(ValueError): validated_sql_name("invalid-name") + @patch("duckdb.connect") @patch("os.makedirs") def test_check(mock_connect, mock_makedirs) -> None: @@ -27,6 +29,7 @@ def test_check(mock_connect, mock_makedirs) -> None: result = destination.check(logger, config) assert result.status == Status.SUCCEEDED + @patch("duckdb.connect") @patch("os.makedirs") def test_check_failure(mock_connect, mock_makedirs) -> None: @@ -38,6 +41,7 @@ def test_check_failure(mock_connect, mock_makedirs) -> None: assert result.status == Status.FAILED assert "Test exception" in result.message + @patch("duckdb.connect") @patch("os.makedirs") def test_write(mock_connect, mock_makedirs) -> None: diff --git a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py index 46d960893fc81..fb040c2ac3cbf 100644 --- a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py @@ -9,14 +9,16 @@ from typing import Any, Dict, Iterable, Mapping, Optional from uuid import uuid4 -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from firebolt.client import DEFAULT_API_URL from firebolt.client.auth import Auth, ClientCredentials, UsernamePassword from firebolt.db import Connection, connect +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type + from .writer import create_firebolt_wirter + logger = getLogger("airbyte") @@ -75,7 +77,6 @@ class DestinationFirebolt(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py index 872db32c38211..397298599d37b 100644 --- a/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firebolt/integration_tests/integration_test.py @@ -9,6 +9,10 @@ from typing import Dict from unittest.mock import MagicMock +from destination_firebolt.destination import DestinationFirebolt, establish_connection +from firebolt.common.exception import FireboltError +from pytest import fixture, mark, raises + from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteStream, @@ -17,9 +21,6 @@ DestinationSyncMode, SyncMode, ) -from destination_firebolt.destination import DestinationFirebolt, establish_connection -from firebolt.common.exception import FireboltError -from pytest import fixture, mark, raises @fixture(scope="module") diff --git a/airbyte-integrations/connectors/destination-firebolt/main.py b/airbyte-integrations/connectors/destination-firebolt/main.py index 1b173be0c2b3e..99c40026fe639 100644 --- a/airbyte-integrations/connectors/destination-firebolt/main.py +++ b/airbyte-integrations/connectors/destination-firebolt/main.py @@ -7,5 +7,6 @@ from destination_firebolt import DestinationFirebolt + if __name__ == "__main__": DestinationFirebolt().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py index d4252f97d5c13..ec74bd77d2852 100644 --- a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py @@ -6,6 +6,9 @@ from typing import Any, Dict from unittest.mock import MagicMock, call, patch +from destination_firebolt.destination import DestinationFirebolt, establish_connection, parse_config +from pytest import fixture + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,8 +20,6 @@ SyncMode, Type, ) -from destination_firebolt.destination import DestinationFirebolt, establish_connection, parse_config -from pytest import fixture @fixture(params=["my_engine", "my_engine.api.firebolt.io"]) @@ -34,6 +35,7 @@ def config(request: Any) -> Dict[str, str]: } return args + @fixture() def legacy_config(): args = { @@ -45,6 +47,7 @@ def legacy_config(): } return args + @fixture def config_external_table() -> Dict[str, str]: args = { diff --git a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py index a063b1efd16c8..7fdd8626e7128 100644 --- a/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-firestore/integration_tests/integration_test.py @@ -7,6 +7,10 @@ from typing import Any, Dict, Mapping import pytest +from destination_firestore import DestinationFirestore +from destination_firestore.writer import FirestoreWriter +from google.cloud import firestore + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,9 +23,6 @@ SyncMode, Type, ) -from destination_firestore import DestinationFirestore -from destination_firestore.writer import FirestoreWriter -from google.cloud import firestore @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-firestore/main.py b/airbyte-integrations/connectors/destination-firestore/main.py index e5bf045a303dc..6f165c46b28dc 100644 --- a/airbyte-integrations/connectors/destination-firestore/main.py +++ b/airbyte-integrations/connectors/destination-firestore/main.py @@ -7,5 +7,6 @@ from destination_firestore import DestinationFirestore + if __name__ == "__main__": DestinationFirestore().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py index 855cba00d92e4..e45411c806be9 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/buffer.py @@ -10,7 +10,6 @@ class WriteBufferMixin: - # Default instance of AirbyteLogger logger = AirbyteLogger() # intervals after which the records_buffer should be cleaned up for selected stream diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py index 41c857f03e3e6..4d0502b81e0da 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/client.py @@ -6,11 +6,13 @@ from typing import Dict import pygsheets -from airbyte_cdk import AirbyteLogger from google.auth.transport.requests import Request from google.oauth2 import credentials as client_account from pygsheets.client import Client as pygsheets_client +from airbyte_cdk import AirbyteLogger + + # the list of required scopes/permissions # more info: https://developers.google.com/sheets/api/guides/authorizing#OAuth2Authorizing SCOPES = [ @@ -20,7 +22,6 @@ class GoogleSheetsClient: - logger = AirbyteLogger() def __init__(self, config: Dict): diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py index 2e1e4343ec55e..f25015f0af754 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py @@ -4,10 +4,11 @@ from typing import Any, Iterable, Mapping +from google.auth.exceptions import RefreshError + from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type -from google.auth.exceptions import RefreshError from .client import GoogleSheetsClient from .helpers import ConnectionTest, get_spreadsheet_id, get_streams_from_catalog @@ -40,7 +41,6 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. """ diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py index 29d27ae744fc7..6cd7390e65a6b 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py @@ -6,11 +6,13 @@ import re from typing import List -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ConfiguredAirbyteCatalog from pygsheets import Spreadsheet, Worksheet from pygsheets.exceptions import WorksheetNotFound +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + STREAMS_COUNT_LIMIT = 200 @@ -39,7 +41,6 @@ def get_streams_from_catalog(catalog: ConfiguredAirbyteCatalog, limit: int = STR class ConnectionTest: - """ Performs connection test write operation to ensure the target spreadsheet is available for writing. Initiating the class itself, performs the connection test and stores the result in ConnectionTest.result property. diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py index 2d2815bf3244c..b6a9ef3814fc6 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/writer.py @@ -3,9 +3,10 @@ # -from airbyte_cdk.models import AirbyteStream from pygsheets import Worksheet +from airbyte_cdk.models import AirbyteStream + from .buffer import WriteBufferMixin from .spreadsheet import GoogleSheets diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py index ce56ac806ac79..200cea5a12d19 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_buffer.py @@ -7,9 +7,11 @@ from typing import Iterable import pytest +from destination_google_sheets.buffer import WriteBufferMixin + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type -from destination_google_sheets.buffer import WriteBufferMixin + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py index a79195eba97d3..caf0a695329ef 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_client.py @@ -8,6 +8,7 @@ from integration_tests.test_helpers import TEST_CONFIG from pygsheets.client import Client as pygsheets_client + # ----- PREPARE ENV ----- # path to configured_catalog json file diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py index d26a46e197faf..e09802ea00576 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_destination.py @@ -7,13 +7,15 @@ from io import StringIO import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import AirbyteConnectionStatus, Status from destination_google_sheets.destination import DestinationGoogleSheets from integration_tests.test_buffer import read_input_messages from integration_tests.test_helpers import TEST_CONFIG from integration_tests.test_writer import TEST_CATALOG, TEST_SPREADSHEET, TEST_STREAM +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import AirbyteConnectionStatus, Status + + # ----- PREPARE ENV ----- @@ -63,7 +65,6 @@ def test_check(): ], ) def test_write(expected, raised): - # clean worksheet after previous test TEST_SPREADSHEET.clean_worksheet(TEST_STREAM) diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py index 3495ff513526d..af98e6ff12d62 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_helpers.py @@ -6,12 +6,14 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_google_sheets.client import GoogleSheetsClient from destination_google_sheets.helpers import ConnectionTest, get_spreadsheet_id, get_streams_from_catalog from destination_google_sheets.spreadsheet import GoogleSheets from pygsheets.client import Client as pygsheets_client +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py index 02e9593e69134..2e60c881104d4 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_spreadsheet.py @@ -9,6 +9,7 @@ from integration_tests.test_helpers import TEST_CONFIG from pygsheets.client import Client as pygsheets_client + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py index e4adbe29eb538..c360221840e98 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/test_writer.py @@ -3,10 +3,12 @@ # import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_google_sheets.writer import GoogleSheetsWriter from integration_tests.test_spreadsheet import TEST_SPREADSHEET +from airbyte_cdk.models import ConfiguredAirbyteCatalog + + # ----- PREPARE ENV ----- diff --git a/airbyte-integrations/connectors/destination-google-sheets/main.py b/airbyte-integrations/connectors/destination-google-sheets/main.py index c2480d5a1f64f..0a94f725540f3 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/main.py +++ b/airbyte-integrations/connectors/destination-google-sheets/main.py @@ -7,5 +7,6 @@ from destination_google_sheets import DestinationGoogleSheets + if __name__ == "__main__": DestinationGoogleSheets().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py index cbbb5c90c0df8..0a23edbb4b624 100644 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py +++ b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py @@ -19,7 +19,6 @@ class DestinationKvdb(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-kvdb/main.py b/airbyte-integrations/connectors/destination-kvdb/main.py index 178789589e5af..09a8a9219af94 100644 --- a/airbyte-integrations/connectors/destination-kvdb/main.py +++ b/airbyte-integrations/connectors/destination-kvdb/main.py @@ -7,5 +7,6 @@ from destination_kvdb import DestinationKvdb + if __name__ == "__main__": DestinationKvdb().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py index 39aa307918a5a..a34b2d738894d 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/run.py @@ -4,9 +4,10 @@ import sys -from airbyte_cdk.entrypoint import launch from destination_meilisearch import DestinationMeilisearch +from airbyte_cdk.entrypoint import launch + def run(): source = DestinationMeilisearch() diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py index c1126786b3b66..06fc951b230f3 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py @@ -4,9 +4,10 @@ from typing import Literal, Optional, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic import BaseModel, Field class UsernamePasswordAuth(BaseModel): diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py index 6f80dc6706c2a..8b390574574c6 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py @@ -16,6 +16,7 @@ from destination_milvus.config import ConfigModel from destination_milvus.indexer import MilvusIndexer + BATCH_SIZE = 128 diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py index bca9df695d8c7..c88294815f4c7 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py @@ -7,13 +7,15 @@ from multiprocessing import Process from typing import Optional +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from destination_milvus.config import MilvusIndexingConfigModel -from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + CLOUD_DEPLOYMENT_MODE = "cloud" diff --git a/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py index 731ba7edbe761..1605cde5b0b0b 100644 --- a/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py +++ b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py @@ -5,14 +5,15 @@ import json import logging -from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_milvus.destination import DestinationMilvus from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Milvus from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status + class MilvusIntegrationTest(BaseIntegrationTest): """ diff --git a/airbyte-integrations/connectors/destination-milvus/main.py b/airbyte-integrations/connectors/destination-milvus/main.py index ed23821ba6c39..5ed2b68506e39 100644 --- a/airbyte-integrations/connectors/destination-milvus/main.py +++ b/airbyte-integrations/connectors/destination-milvus/main.py @@ -7,5 +7,6 @@ from destination_milvus import DestinationMilvus + if __name__ == "__main__": DestinationMilvus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py index 982c42d1fa86b..4ce5784f760ea 100644 --- a/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_milvus.config import ConfigModel from destination_milvus.destination import DestinationMilvus +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationMilvus(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py index f88d064862c3f..3087226daf94a 100644 --- a/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py @@ -6,11 +6,12 @@ import unittest from unittest.mock import Mock, call, patch -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode from destination_milvus.config import MilvusIndexingConfigModel, NoAuth, TokenAuth from destination_milvus.indexer import MilvusIndexer from pymilvus import DataType +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode + @patch("destination_milvus.indexer.connections") @patch("destination_milvus.indexer.utility") diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py index 879c7a05c2c96..08bdbfa8e56a4 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/destination.py @@ -15,6 +15,9 @@ from urllib.parse import urlparse import orjson +from serpyco_rs import Serializer +from typing_extensions import override + from airbyte_cdk import AirbyteStream, ConfiguredAirbyteStream, SyncMode from airbyte_cdk.destinations import Destination from airbyte_cdk.exception_handler import init_uncaught_exception_handler @@ -35,8 +38,7 @@ from airbyte_cdk.sql.types import SQLTypeConverter from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor from destination_motherduck.processors.motherduck import MotherDuckConfig, MotherDuckSqlProcessor -from serpyco_rs import Serializer -from typing_extensions import override + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py index a37cb35ebbcff..c25bc81cbdfa8 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/duckdb.py @@ -10,17 +10,19 @@ from urllib.parse import parse_qsl, urlparse import pyarrow as pa -from airbyte_cdk import DestinationSyncMode -from airbyte_cdk.sql import exceptions as exc -from airbyte_cdk.sql.constants import AB_EXTRACTED_AT_COLUMN, DEBUG_MODE -from airbyte_cdk.sql.secrets import SecretString -from airbyte_cdk.sql.shared.sql_processor import SqlConfig, SqlProcessorBase, SQLRuntimeError from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from sqlalchemy import Executable, TextClause, create_engine, text from sqlalchemy.exc import ProgrammingError, SQLAlchemyError +from airbyte_cdk import DestinationSyncMode +from airbyte_cdk.sql import exceptions as exc +from airbyte_cdk.sql.constants import AB_EXTRACTED_AT_COLUMN, DEBUG_MODE +from airbyte_cdk.sql.secrets import SecretString +from airbyte_cdk.sql.shared.sql_processor import SqlConfig, SqlProcessorBase, SQLRuntimeError + + if TYPE_CHECKING: from sqlalchemy.engine import Connection, Engine diff --git a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py index d05a75a56a7c1..6da7f84176747 100644 --- a/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py +++ b/airbyte-integrations/connectors/destination-motherduck/destination_motherduck/processors/motherduck.py @@ -18,14 +18,16 @@ import warnings -from airbyte_cdk.sql.constants import DEBUG_MODE -from airbyte_cdk.sql.secrets import SecretString -from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor from duckdb_engine import DuckDBEngineWarning from overrides import overrides from pydantic import Field from sqlalchemy import Engine, create_engine +from airbyte_cdk.sql.constants import DEBUG_MODE +from airbyte_cdk.sql.secrets import SecretString +from destination_motherduck.processors.duckdb import DuckDBConfig, DuckDBSqlProcessor + + # Suppress warnings from DuckDB about reflection on indices. # https://github.com/Mause/duckdb_engine/issues/905 warnings.filterwarnings( diff --git a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py index 076bc70146854..9ae7e741e9bd1 100644 --- a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py @@ -14,6 +14,10 @@ import duckdb import pytest +from destination_motherduck import DestinationMotherDuck +from destination_motherduck.destination import CONFIG_MOTHERDUCK_API_KEY +from faker import Faker + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -27,14 +31,10 @@ Type, ) from airbyte_cdk.sql.secrets import SecretString -from destination_motherduck import DestinationMotherDuck -from destination_motherduck.destination import CONFIG_MOTHERDUCK_API_KEY -from faker import Faker + CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +45,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -89,9 +87,7 @@ def disable_destination_modification(monkeypatch, request): if "disable_autouse" in request.keywords: return else: - monkeypatch.setattr( - DestinationMotherDuck, "_get_destination_path", lambda _, x: x - ) + monkeypatch.setattr(DestinationMotherDuck, "_get_destination_path", lambda _, x: x) @pytest.fixture(scope="module") @@ -254,9 +250,7 @@ def airbyte_message2_update(airbyte_message2: AirbyteMessage, test_table_name: s @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.fixture @@ -308,11 +302,7 @@ def _state(data: Dict[str, Any]) -> AirbyteMessage: @pytest.fixture() -def sql_processor( - configured_catalogue, - test_schema_name, - config: Dict[str, str] - ): +def sql_processor(configured_catalogue, test_schema_name, config: Dict[str, str]): destination = DestinationMotherDuck() path = config.get("destination_path", "md:") if CONFIG_MOTHERDUCK_API_KEY in config: @@ -320,7 +310,7 @@ def sql_processor( configured_catalog=configured_catalogue, schema_name=test_schema_name, db_path=path, - motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY] + motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY], ) else: processor = destination._get_sql_processor( @@ -349,13 +339,7 @@ def test_write( generator = destination.write( config, configured_catalogue, - [ - airbyte_message1, - airbyte_message2, - airbyte_message3, - airbyte_message4, - airbyte_message5 - ], + [airbyte_message1, airbyte_message2, airbyte_message3, airbyte_message4, airbyte_message5], ) result = list(generator) @@ -407,9 +391,7 @@ def test_write_dupe( assert sql_result[1][1] == "777-54-0664" -def _airbyte_messages( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) @@ -431,9 +413,7 @@ def _airbyte_messages( yield message -def _airbyte_messages_with_inconsistent_json_fields( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) random.seed(0) @@ -453,25 +433,19 @@ def _airbyte_messages_with_inconsistent_json_fields( data=( { "key1": fake.unique.name(), - "key2": str(fake.ssn()) - if random.random() < 0.5 - else str(random.randrange(1000, 9999999999999)), + "key2": str(fake.ssn()) if random.random() < 0.5 else str(random.randrange(1000, 9999999999999)), "nested1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": str(fake.ssn()) - if random.random() < 0.5 - else random.randrange(1000, 9999999999999), + "key4": str(fake.ssn()) if random.random() < 0.5 else random.randrange(1000, 9999999999999), "dictionary1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": "True" - if random.random() < 0.5 - else True, + "key4": "True" if random.random() < 0.5 else True, } ), } @@ -517,16 +491,11 @@ def test_large_number_of_writes( generator = destination.write( config, configured_catalogue, - airbyte_message_generator( - TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name - ), + airbyte_message_generator(TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name), ) result = list(generator) assert len(result) == TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) - sql_result = sql_processor._execute_sql( - "SELECT count(1) " - f"FROM {test_schema_name}.{test_large_table_name}" - ) + sql_result = sql_processor._execute_sql("SELECT count(1) " f"FROM {test_schema_name}.{test_large_table_name}") assert sql_result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-motherduck/main.py b/airbyte-integrations/connectors/destination-motherduck/main.py index 710f67455a47e..bc605551741dc 100644 --- a/airbyte-integrations/connectors/destination-motherduck/main.py +++ b/airbyte-integrations/connectors/destination-motherduck/main.py @@ -5,5 +5,6 @@ from destination_motherduck.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py index 1245d9785f015..c062b05670530 100644 --- a/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-motherduck/unit_tests/destination_unit_tests.py @@ -5,9 +5,10 @@ from unittest.mock import Mock, patch import pytest -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from destination_motherduck.destination import CONFIG_DEFAULT_SCHEMA, DestinationMotherDuck, validated_sql_name +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + def test_validated_sql_name() -> None: assert validated_sql_name("valid_name") == "valid_name" diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py index 67e610934d435..54cc7631fb682 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py @@ -57,13 +57,17 @@ def get_configured_stream_info( ) matching_streams: list[ConfiguredAirbyteStream] = [ - stream for stream in self.configured_catalog.streams if stream.stream.name == stream_name + stream + for stream in self.configured_catalog.streams + if stream.stream.name == stream_name ] if not matching_streams: raise exc.AirbyteStreamNotFoundError( stream_name=stream_name, context={ - "available_streams": [stream.stream.name for stream in self.configured_catalog.streams], + "available_streams": [ + stream.stream.name for stream in self.configured_catalog.streams + ], }, ) diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py index 74bad59eca548..707ed7d4ed0d5 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py @@ -16,13 +16,22 @@ from airbyte import exceptions as exc from airbyte.strategies import WriteStrategy -from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, Type +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStreamState, + Type, +) + from destination_pgvector.common.state.state_writers import StateWriterBase, StdOutStateWriter if TYPE_CHECKING: from collections.abc import Iterable, Iterator from airbyte._batch_handles import BatchHandle + from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.common.state.state_writers import StateWriterBase @@ -224,7 +233,9 @@ def process_airbyte_messages( stream_name = record_msg.stream if stream_name not in stream_schemas: - stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema(stream_name=stream_name) + stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema( + stream_name=stream_name + ) self.process_record_message( record_msg, diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py index a297a01948469..9b5af54bf2869 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py @@ -21,13 +21,14 @@ from airbyte.strategies import WriteStrategy from airbyte.types import SQLTypeConverter from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode -from destination_pgvector.common.destinations.record_processor import RecordProcessorBase -from destination_pgvector.common.state.state_writers import StdOutStateWriter from pandas import Index from pydantic import BaseModel from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update from sqlalchemy.sql.elements import TextClause +from destination_pgvector.common.destinations.record_processor import RecordProcessorBase +from destination_pgvector.common.state.state_writers import StdOutStateWriter + if TYPE_CHECKING: from collections.abc import Generator @@ -35,14 +36,15 @@ from airbyte._processors.file.base import FileWriterBase from airbyte.secrets.base import SecretString from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage - from destination_pgvector.common.catalog.catalog_providers import CatalogProvider - from destination_pgvector.common.state.state_writers import StateWriterBase from sqlalchemy.engine import Connection, Engine from sqlalchemy.engine.cursor import CursorResult from sqlalchemy.engine.reflection import Inspector from sqlalchemy.sql.base import Executable from sqlalchemy.sql.type_api import TypeEngine + from destination_pgvector.common.catalog.catalog_providers import CatalogProvider + from destination_pgvector.common.state.state_writers import StateWriterBase + class RecordDedupeMode(enum.Enum): APPEND = "append" @@ -101,7 +103,9 @@ def get_vendor_client(self) -> object: Raises `NotImplementedError` if a custom vendor client is not defined. """ - raise NotImplementedError(f"The type '{type(self).__name__}' does not define a custom client.") + raise NotImplementedError( + f"The type '{type(self).__name__}' does not define a custom client." + ) class SqlProcessorBase(RecordProcessorBase): @@ -265,7 +269,9 @@ def _get_table_by_name( query. To ignore the cache and force a refresh, set 'force_refresh' to True. """ if force_refresh and shallow_okay: - raise exc.PyAirbyteInternalError(message="Cannot force refresh and use shallow query at the same time.") + raise exc.PyAirbyteInternalError( + message="Cannot force refresh and use shallow query at the same time." + ) if force_refresh and table_name in self._cached_table_definitions: self._invalidate_table_cache(table_name) @@ -306,7 +312,9 @@ def _ensure_schema_exists( if DEBUG_MODE: found_schemas = self._get_schemas_list() - assert schema_name in found_schemas, f"Schema {schema_name} was not created. Found: {found_schemas}" + assert schema_name in found_schemas, ( + f"Schema {schema_name} was not created. Found: {found_schemas}" + ) def _quote_identifier(self, identifier: str) -> str: """Return the given identifier, quoted.""" @@ -365,7 +373,8 @@ def _get_schemas_list( return [ found_schema.split(".")[-1].strip('"') for found_schema in found_schemas - if "." not in found_schema or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) + if "." not in found_schema + or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) ] def _ensure_final_table_exists( @@ -522,7 +531,9 @@ def finalizing_batches( Returns a mapping of batch IDs to batch handles, for those processed batches. """ batches_to_finalize: list[BatchHandle] = self.file_writer.get_pending_batches(stream_name) - state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[stream_name].copy() + state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[ + stream_name + ].copy() self._pending_state_messages[stream_name].clear() progress.log_batches_finalizing(stream_name, len(batches_to_finalize)) @@ -581,7 +592,9 @@ def _write_files_to_new_table( for file_path in files: dataframe = pd.read_json(file_path, lines=True) - sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions(stream_name) + sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions( + stream_name + ) # Remove fields that are not in the schema for col_name in dataframe.columns: @@ -619,7 +632,10 @@ def _add_column_to_table( ) -> None: """Add a column to the given table.""" self._execute_sql( - text(f"ALTER TABLE {self._fully_qualified(table.name)} " f"ADD COLUMN {column_name} {column_type}"), + text( + f"ALTER TABLE {self._fully_qualified(table.name)} " + f"ADD COLUMN {column_name} {column_type}" + ), ) def _add_missing_columns_to_table( @@ -677,7 +693,9 @@ def _write_temp_table_to_final_table( ) if write_strategy == WriteStrategy.AUTO: - configured_destination_sync_mode: DestinationSyncMode = self.catalog_provider.get_destination_sync_mode(stream_name) + configured_destination_sync_mode: DestinationSyncMode = ( + self.catalog_provider.get_destination_sync_mode(stream_name) + ) if configured_destination_sync_mode == DestinationSyncMode.overwrite: write_strategy = WriteStrategy.REPLACE elif configured_destination_sync_mode == DestinationSyncMode.append: @@ -802,13 +820,11 @@ def _swap_temp_table_with_final_table( _ = stream_name deletion_name = f"{final_table_name}_deleteme" - commands = "\n".join( - [ - f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", - f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " f"TO {final_table_name};", - f"DROP TABLE {self._fully_qualified(deletion_name)};", - ] - ) + commands = "\n".join([ + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME TO {final_table_name};", + f"DROP TABLE {self._fully_qualified(deletion_name)};", + ]) self._execute_sql(commands) def _merge_temp_table_to_final_table( @@ -882,16 +898,23 @@ def _emulated_merge_temp_table_to_final_table( temp_table = self._get_table_by_name(temp_table_name) pk_columns = self._get_primary_keys(stream_name) - columns_to_update: set[str] = self._get_sql_column_definitions(stream_name=stream_name).keys() - set(pk_columns) + columns_to_update: set[str] = self._get_sql_column_definitions( + stream_name=stream_name + ).keys() - set(pk_columns) # Create a dictionary mapping columns in users_final to users_stage for updating update_values = { - self._get_column_by_name(final_table, column): (self._get_column_by_name(temp_table, column)) for column in columns_to_update + self._get_column_by_name(final_table, column): ( + self._get_column_by_name(temp_table, column) + ) + for column in columns_to_update } # Craft the WHERE clause for composite primary keys join_conditions = [ - self._get_column_by_name(final_table, pk_column) == self._get_column_by_name(temp_table, pk_column) for pk_column in pk_columns + self._get_column_by_name(final_table, pk_column) + == self._get_column_by_name(temp_table, pk_column) + for pk_column in pk_columns ] join_clause = and_(*join_conditions) @@ -906,7 +929,9 @@ def _emulated_merge_temp_table_to_final_table( where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null() # Select records from temp_table that are not in final_table - select_new_records_stmt = select([temp_table]).select_from(joined_table).where(where_not_exists_clause) + select_new_records_stmt = ( + select([temp_table]).select_from(joined_table).where(where_not_exists_clause) + ) # Craft the INSERT statement using the select statement insert_new_records_stmt = insert(final_table).from_select( diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py index d3e72198d82e3..eb6ebdc51c38a 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/config.py @@ -22,7 +22,6 @@ class Config: class PGVectorIndexingModel(BaseModel): - host: str = Field( ..., title="Host", diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py index 32ae5291efdfa..44a6abbb7b2c2 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py @@ -19,6 +19,7 @@ DestinationSyncMode, Status, ) + from destination_pgvector import pgvector_processor from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.config import ConfigModel @@ -29,7 +30,9 @@ class DestinationPGVector(Destination): sql_processor: pgvector_processor.PGVectorProcessor - def _init_sql_processor(self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None) -> None: + def _init_sql_processor( + self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None + ) -> None: self.sql_processor = pgvector_processor.PGVectorProcessor( sql_config=pgvector_processor.PostgresConfig( host=config.indexing.host, @@ -67,7 +70,9 @@ def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionS self.sql_processor.sql_config.connect() return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") + return AirbyteConnectionStatus( + status=Status.FAILED, message=f"An exception occurred: {repr(e)}" + ) def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: return ConnectorSpecification( diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py index 5757951e05d14..5a33f25ff59bf 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py @@ -13,16 +13,27 @@ from airbyte._processors.file.jsonl import JsonlWriter from airbyte.secrets import SecretString from airbyte_cdk.destinations.vector_db_based import embedder -from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor as DocumentSplitter -from airbyte_cdk.destinations.vector_db_based.document_processor import ProcessingConfigModel as DocumentSplitterConfig +from airbyte_cdk.destinations.vector_db_based.document_processor import ( + DocumentProcessor as DocumentSplitter, +) +from airbyte_cdk.destinations.vector_db_based.document_processor import ( + ProcessingConfigModel as DocumentSplitterConfig, +) from airbyte_cdk.models import AirbyteRecordMessage -from destination_pgvector.common.catalog.catalog_providers import CatalogProvider -from destination_pgvector.common.sql.sql_processor import SqlConfig, SqlProcessorBase -from destination_pgvector.globals import CHUNK_ID_COLUMN, DOCUMENT_CONTENT_COLUMN, DOCUMENT_ID_COLUMN, EMBEDDING_COLUMN, METADATA_COLUMN from overrides import overrides from pgvector.sqlalchemy import Vector from typing_extensions import Protocol +from destination_pgvector.common.catalog.catalog_providers import CatalogProvider +from destination_pgvector.common.sql.sql_processor import SqlConfig, SqlProcessorBase +from destination_pgvector.globals import ( + CHUNK_ID_COLUMN, + DOCUMENT_CONTENT_COLUMN, + DOCUMENT_ID_COLUMN, + EMBEDDING_COLUMN, + METADATA_COLUMN, +) + class PostgresConfig(SqlConfig): """Configuration for the Postgres cache. @@ -39,7 +50,9 @@ class PostgresConfig(SqlConfig): @overrides def get_sql_alchemy_url(self) -> SecretString: """Return the SQLAlchemy URL to use.""" - return SecretString(f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}") + return SecretString( + f"postgresql+psycopg2://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + ) @overrides def get_database_name(self) -> str: @@ -123,7 +136,9 @@ def _emulated_merge_temp_table_to_final_table( So instead of using UPDATE and then INSERT, we will DELETE all rows for included primary keys and then call the append implementation to insert new rows. """ - columns_list: list[str] = list(self._get_sql_column_definitions(stream_name=stream_name).keys()) + columns_list: list[str] = list( + self._get_sql_column_definitions(stream_name=stream_name).keys() + ) delete_statement = dedent( f""" diff --git a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py index def5bb2988018..2046e0945847e 100644 --- a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py @@ -10,11 +10,11 @@ import psycopg2 from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest from airbyte_cdk.models import DestinationSyncMode, Status + from destination_pgvector.destination import DestinationPGVector class PGVectorIntegrationTest(BaseIntegrationTest): - def setUp(self): with open("secrets/config.json", "r") as f: self.config = json.loads(f.read()) @@ -331,7 +331,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationPGVector() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -358,4 +358,3 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["document_content"] == "Dogs are" assert third_written_record["document_id"] == "Stream_mystream_Key_0" assert third_written_record["document_content"] == "number 0" - diff --git a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py index 0b671c9ff1aaa..0b48b176d7ae4 100644 --- a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py @@ -8,6 +8,7 @@ from airbyte.strategies import WriteStrategy from airbyte_cdk.models import ConnectorSpecification, Status + from destination_pgvector.config import ConfigModel from destination_pgvector.destination import DestinationPGVector diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py index 38ade8d0834a0..1256143155d50 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class PineconeIndexingModel(BaseModel): pinecone_key: str = Field( diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py index 9d199b3ce50b8..45af5865a9c63 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py @@ -17,6 +17,7 @@ from destination_pinecone.config import ConfigModel from destination_pinecone.indexer import PineconeIndexer + BATCH_SIZE = 32 diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py index b2d67010192e3..8777829993cfd 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py @@ -7,14 +7,16 @@ from typing import Optional import urllib3 +from pinecone import PineconeException +from pinecone.grpc import PineconeGRPC + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception from airbyte_cdk.models import AirbyteConnectionStatus, Status from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog, DestinationSyncMode from destination_pinecone.config import PineconeIndexingModel -from pinecone import PineconeException -from pinecone.grpc import PineconeGRPC + # large enough to speed up processing, small enough to not hit pinecone request limits PINECONE_BATCH_SIZE = 40 diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py index 46215e878464d..b51c3ba18131d 100644 --- a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py @@ -7,6 +7,13 @@ import os import time +from destination_pinecone.destination import DestinationPinecone +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Pinecone +from pinecone import Pinecone as PineconeREST +from pinecone import PineconeException +from pinecone.grpc import PineconeGRPC + from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest from airbyte_cdk.models import ( @@ -21,12 +28,6 @@ SyncMode, Type, ) -from destination_pinecone.destination import DestinationPinecone -from langchain.embeddings import OpenAIEmbeddings -from langchain.vectorstores import Pinecone -from pinecone import Pinecone as PineconeREST -from pinecone import PineconeException -from pinecone.grpc import PineconeGRPC class PineconeIntegrationTest(BaseIntegrationTest): @@ -35,14 +36,14 @@ def _init_pinecone(self): self.pinecone_index = self.pc.Index(self.config["indexing"]["index"]) self.pc_rest = PineconeREST(api_key=self.config["indexing"]["pinecone_key"]) self.pinecone_index_rest = self.pc_rest.Index(name=self.config["indexing"]["index"]) - + def _wait(self): - print("Waiting for Pinecone...", end='', flush=True) + print("Waiting for Pinecone...", end="", flush=True) for i in range(15): time.sleep(1) - print(".", end='', flush=True) + print(".", end="", flush=True) print() # Move to the next line after the loop - + def setUp(self): with open("secrets/config.json", "r") as f: self.config = json.loads(f.read()) @@ -50,28 +51,28 @@ def setUp(self): def tearDown(self): self._wait() - # make sure pinecone is initialized correctly before cleaning up + # make sure pinecone is initialized correctly before cleaning up self._init_pinecone() try: self.pinecone_index.delete(delete_all=True) except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in default namespace. No data in the index/namespace.") try: self.pinecone_index.delete(delete_all=True, namespace="ns1") except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in ns1 namespace. No data in the index/namespace.") def test_integration_test_flag_is_set(self): assert "PYTEST_CURRENT_TEST" in os.environ def test_check_valid_config(self): - outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) + outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): @@ -88,7 +89,7 @@ def test_check_invalid_config(self): }, }, ) - + assert outcome.status == Status.FAILED def test_write(self): @@ -99,21 +100,20 @@ def test_write(self): # initial sync destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - - - self._wait() + + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 # incrementalally update a doc incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) - - self._wait() - + + self._wait() + result = self.pinecone_index.query( vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_ab_record_id": "mystream_2"}, include_metadata=True ) - + assert len(result.matches) == 1 assert ( result.matches[0].metadata["text"] == "str_col: Cats are nice" @@ -135,19 +135,21 @@ def test_write_with_namespace(self): destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - self._wait() + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 - def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}} + stream_schema = { + "type": "object", + "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}, + } overwrite_stream = ConfiguredAirbyteStream( stream=AirbyteStream( - name="mystream", + name="mystream", namespace="ns1", - json_schema=stream_schema, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + json_schema=stream_schema, + supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], ), primary_key=[["int_col"]], sync_mode=SyncMode.incremental, @@ -155,14 +157,9 @@ def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSy ) return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) - + def _record_with_namespace(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, - namespace="ns1", - data={"str_col": str_value, "int_col": int_value}, - emitted_at=0) + type=Type.RECORD, + record=AirbyteRecordMessage(stream=stream, namespace="ns1", data={"str_col": str_value, "int_col": int_value}, emitted_at=0), ) - - - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-pinecone/main.py b/airbyte-integrations/connectors/destination-pinecone/main.py index 9a12601a2cfa6..fdc7a3e630993 100644 --- a/airbyte-integrations/connectors/destination-pinecone/main.py +++ b/airbyte-integrations/connectors/destination-pinecone/main.py @@ -7,5 +7,6 @@ from destination_pinecone import DestinationPinecone + if __name__ == "__main__": DestinationPinecone().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py index 3145b0a66e8c1..bff3f91507c14 100644 --- a/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py +++ b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py @@ -10,6 +10,7 @@ from langchain.llms import OpenAI from langchain.vectorstores import Pinecone + # Run with OPENAI_API_KEY, PINECONE_KEY and PINECONE_ENV set in the environment embeddings = OpenAIEmbeddings() diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py index e62c0bb038e9a..3a0329deee083 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_pinecone.config import ConfigModel from destination_pinecone.destination import DestinationPinecone +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationPinecone(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py index 5d4b714a79024..3fb46fa23a8fc 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py @@ -7,23 +7,24 @@ import pytest import urllib3 -from airbyte_cdk.models import ConfiguredAirbyteCatalog from destination_pinecone.config import PineconeIndexingModel from destination_pinecone.indexer import PineconeIndexer from pinecone import IndexDescription, exceptions from pinecone.grpc import PineconeGRPC from pinecone.models import IndexList +from airbyte_cdk.models import ConfiguredAirbyteCatalog + def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") - with patch.object(PineconeGRPC, 'Index') as mock_index: + with patch.object(PineconeGRPC, "Index") as mock_index: indexer = PineconeIndexer(config, 3) - + indexer.pc.list_indexes = MagicMock() indexer.pc.list_indexes.return_value.indexes = create_mock_list_indexes() - + indexer.pc.describe_index = MagicMock() if side_effect: indexer.pc.describe_index.side_effect = side_effect @@ -31,6 +32,7 @@ def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): indexer.pc.describe_index.return_value = create_index_description(dimensions=embedding_dimensions) return indexer + def create_index_description(dimensions=3, pod_type="p1"): return IndexDescription( name="", @@ -41,18 +43,21 @@ def create_index_description(dimensions=3, pod_type="p1"): status=None, ) + def create_mock_list_indexes(): return [{"name": "myindex"}, {"name": "myindex2"}] + @pytest.fixture(scope="module", autouse=True) def mock_describe_index(): with patch("pinecone.describe_index") as mock: mock.return_value = create_index_description() yield mock + @pytest.fixture(scope="module", autouse=True) def mock_determine_spec_type(): - with patch.object(PineconeIndexer, 'determine_spec_type') as mock: + with patch.object(PineconeIndexer, "determine_spec_type") as mock: mock.return_value = "pod" yield mock @@ -77,7 +82,7 @@ def test_get_source_tag_with_pytest(): @patch.dict("os.environ", {"RUN_IN_AIRBYTE_CI": "Value does not matter"}) def test_get_source_tag_with_ci(): - # CI and pytest is running + # CI and pytest is running indexer = create_pinecone_indexer() assert indexer.get_source_tag() == "airbyte_test" @@ -143,6 +148,7 @@ def test_pinecone_index_upsert_and_delete_starter(mock_describe_index, mock_dete namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "pod" @@ -160,9 +166,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(filter={'_ab_record_id': {'$in': ['delete_id1', 'delete_id2']}}, namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}}, namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -173,6 +177,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "serverless" @@ -190,9 +195,7 @@ def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_d "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(ids=['delete_id1', 'delete_id2'], namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(ids=["delete_id1", "delete_id2"], namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -311,13 +314,12 @@ def test_pinecone_pre_sync_starter(mock_describe_index, mock_determine_spec_type ("myindex", None, 3, True, None), ("other_index", None, 3, False, "Index other_index does not exist in environment"), ( - "myindex", + "myindex", urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve 'controller.myenv.pinecone.io'")), 3, False, "Failed to resolve environment", - - ), + ), ("myindex", exceptions.UnauthorizedException(http_resp=urllib3.HTTPResponse(body="No entry!")), 3, False, "No entry!"), ("myindex", None, 4, False, "Make sure embedding and indexing configurations match."), ("myindex", Exception("describe failed"), 3, False, "describe failed"), diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py index e877a93bcf6d0..31102ed2d8581 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py @@ -5,9 +5,10 @@ from typing import Literal, Union -from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel from pydantic.v1 import BaseModel, Field +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel + class NoAuth(BaseModel): mode: Literal["no_auth"] = Field("no_auth", const=True) diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py index 8f2756c17b208..bde7a6791044e 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py @@ -15,6 +15,7 @@ from destination_qdrant.config import ConfigModel from destination_qdrant.indexer import QdrantIndexer + BATCH_SIZE = 256 @@ -29,7 +30,6 @@ def _init_indexer(self, config: ConfigModel): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - config_model = ConfigModel.parse_obj(config) self._init_indexer(config_model) writer = Writer( @@ -48,7 +48,6 @@ def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteCon return AirbyteConnectionStatus(status=Status.SUCCEEDED) def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: - return ConnectorSpecification( documentationUrl="https://docs.airbyte.com/integrations/destinations/qdrant", supportsIncremental=True, diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py index 1d78d8730e7a6..40292a02f991a 100644 --- a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py @@ -6,15 +6,17 @@ import uuid from typing import List, Optional +from qdrant_client import QdrantClient, models +from qdrant_client.conversions.common_types import PointsSelector +from qdrant_client.models import Distance, PayloadSchemaType, VectorParams + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog, Level, Type from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from destination_qdrant.config import QdrantIndexingConfigModel -from qdrant_client import QdrantClient, models -from qdrant_client.conversions.common_types import PointsSelector -from qdrant_client.models import Distance, PayloadSchemaType, VectorParams + DISTANCE_METRIC_MAP = { "dot": Distance.DOT, diff --git a/airbyte-integrations/connectors/destination-qdrant/main.py b/airbyte-integrations/connectors/destination-qdrant/main.py index 42c2e8492e9fe..003ce287d2637 100644 --- a/airbyte-integrations/connectors/destination-qdrant/main.py +++ b/airbyte-integrations/connectors/destination-qdrant/main.py @@ -7,5 +7,6 @@ from destination_qdrant import DestinationQdrant + if __name__ == "__main__": DestinationQdrant().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py index ec3cb89da4c29..7ffd02511229f 100644 --- a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_qdrant.config import ConfigModel from destination_qdrant.destination import DestinationQdrant +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationQdrant(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py index eef6619302aa2..34c83e028b051 100644 --- a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py @@ -5,12 +5,13 @@ import unittest from unittest.mock import Mock, call -from airbyte_cdk.destinations.vector_db_based.utils import format_exception -from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, AirbyteStream, DestinationSyncMode, Level, SyncMode, Type from destination_qdrant.config import QdrantIndexingConfigModel from destination_qdrant.indexer import QdrantIndexer from qdrant_client import models +from airbyte_cdk.destinations.vector_db_based.utils import format_exception +from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, AirbyteStream, DestinationSyncMode, Level, SyncMode, Type + class TestQdrantIndexer(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py b/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py index 5a7512f1ae149..ff26ffef0e49d 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/destination_rabbitmq/destination.py @@ -8,11 +8,13 @@ from typing import Any, Iterable, Mapping import pika -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from pika.adapters.blocking_connection import BlockingConnection from pika.spec import BasicProperties +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + + _DEFAULT_PORT = 5672 diff --git a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py index f99c64178d4f9..edff223285066 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/integration_tests/integration_test.py @@ -5,6 +5,8 @@ import json from unittest.mock import Mock +from destination_rabbitmq.destination import DestinationRabbitmq, create_connection + from airbyte_cdk.models import AirbyteMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, @@ -15,7 +17,7 @@ DestinationSyncMode, SyncMode, ) -from destination_rabbitmq.destination import DestinationRabbitmq, create_connection + TEST_STREAM = "animals" TEST_NAMESPACE = "test_namespace" diff --git a/airbyte-integrations/connectors/destination-rabbitmq/main.py b/airbyte-integrations/connectors/destination-rabbitmq/main.py index fc09374015c71..cb9d8cbf44dfe 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/main.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/main.py @@ -7,5 +7,6 @@ from destination_rabbitmq import DestinationRabbitmq + if __name__ == "__main__": DestinationRabbitmq().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py index 57c34b6f9f580..23c4d6feeae41 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-rabbitmq/unit_tests/unit_test.py @@ -7,6 +7,9 @@ from unittest import mock from unittest.mock import Mock +from destination_rabbitmq.destination import DestinationRabbitmq +from pika.spec import Queue + from airbyte_cdk.models import AirbyteMessage, Status, Type from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, @@ -17,8 +20,7 @@ DestinationSyncMode, SyncMode, ) -from destination_rabbitmq.destination import DestinationRabbitmq -from pika.spec import Queue + config = { "host": "test.rabbitmq", diff --git a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py index a08452ebc2bd6..87a07ad189c24 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py +++ b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py @@ -20,7 +20,6 @@ def write( configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage], ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py index a78f9b391aa69..3053118858efc 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sftp-json/integration_tests/integration_test.py @@ -9,6 +9,9 @@ import docker import pytest +from destination_sftp_json import DestinationSftpJson +from destination_sftp_json.client import SftpClient + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import ( AirbyteMessage, @@ -22,8 +25,6 @@ SyncMode, Type, ) -from destination_sftp_json import DestinationSftpJson -from destination_sftp_json.client import SftpClient @pytest.fixture(scope="module") diff --git a/airbyte-integrations/connectors/destination-sftp-json/main.py b/airbyte-integrations/connectors/destination-sftp-json/main.py index 84167715983d5..6bcd0cbf13686 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/main.py +++ b/airbyte-integrations/connectors/destination-sftp-json/main.py @@ -7,5 +7,6 @@ from destination_sftp_json import DestinationSftpJson + if __name__ == "__main__": DestinationSftpJson().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-sftp-json/setup.py b/airbyte-integrations/connectors/destination-sftp-json/setup.py index 1ade4c13054a7..00c14d35802d3 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/setup.py +++ b/airbyte-integrations/connectors/destination-sftp-json/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = ["airbyte-cdk", "smart_open==5.1.0", "paramiko==2.10.1"] TEST_REQUIREMENTS = ["pytest~=6.1", "docker==5.0.3"] diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py index 2867def1a4a99..cdf0670f7dd28 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py @@ -330,9 +330,9 @@ def _ensure_schema_exists( if DEBUG_MODE: found_schemas = self._get_schemas_list() - assert ( - schema_name in found_schemas - ), f"Schema {schema_name} was not created. Found: {found_schemas}" + assert schema_name in found_schemas, ( + f"Schema {schema_name} was not created. Found: {found_schemas}" + ) def _quote_identifier(self, identifier: str) -> str: """Return the given identifier, quoted.""" @@ -839,9 +839,8 @@ def _swap_temp_table_with_final_table( _ = stream_name deletion_name = f"{final_table_name}_deleteme" commands = "\n".join([ - f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", - f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " - f"TO {final_table_name};", + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME TO {final_table_name};", f"DROP TABLE {self._fully_qualified(deletion_name)};", ]) self._execute_sql(commands) diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py index 769eb67fc9849..3a514b274173c 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/state/state_writers.py @@ -7,7 +7,6 @@ import abc from typing import TYPE_CHECKING - if TYPE_CHECKING: from airbyte_protocol.models.airbyte_protocol import AirbyteStateMessage diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py index 3148de0862d1c..17a602da056bd 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/globals.py @@ -3,7 +3,6 @@ from __future__ import annotations - DOCUMENT_ID_COLUMN = "document_id" CHUNK_ID_COLUMN = "chunk_id" METADATA_COLUMN = "metadata" diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py index 864c227d28683..1976d878dae70 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py @@ -337,7 +337,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationSnowflakeCortex() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -364,8 +364,6 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["DOCUMENT_CONTENT"] == '"Dogs are"' assert third_written_record["DOCUMENT_ID"] == "Stream_mystream_Key_0" assert third_written_record["DOCUMENT_CONTENT"] == '"number 0"' - - """ Following tests are not code specific, but are useful to confirm that the Cortex functions are available and behaving as expcected diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py index e5595ecf88916..a89fb499be9c4 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/unit_tests/destination_test.py @@ -2,11 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import unittest from unittest.mock import MagicMock, Mock, patch from airbyte.strategies import WriteStrategy -import logging from airbyte_cdk.models import ConnectorSpecification, Status from destination_snowflake_cortex.config import ConfigModel diff --git a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py index f8049536e7b42..c1466fbf1ba6e 100644 --- a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py +++ b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py @@ -37,7 +37,6 @@ def _get_destination_path(destination_path: str) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. @@ -65,9 +64,7 @@ def write( # delete the tables query = """ DROP TABLE IF EXISTS {} - """.format( - table_name - ) + """.format(table_name) con.execute(query) # create the table if needed query = """ @@ -76,9 +73,7 @@ def write( _airbyte_emitted_at TEXT, _airbyte_data TEXT ) - """.format( - table_name=table_name - ) + """.format(table_name=table_name) con.execute(query) buffer = defaultdict(list) @@ -87,13 +82,10 @@ def write( if message.type == Type.STATE: # flush the buffer for stream_name in buffer.keys(): - query = """ INSERT INTO {table_name} VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) + """.format(table_name=f"_airbyte_raw_{stream_name}") con.executemany(query, buffer[stream_name]) @@ -113,13 +105,10 @@ def write( # flush any remaining messages for stream_name in buffer.keys(): - query = """ INSERT INTO {table_name} VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) + """.format(table_name=f"_airbyte_raw_{stream_name}") con.executemany(query, buffer[stream_name]) diff --git a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py index ed5d04c99edaa..52ea5544e5ba3 100644 --- a/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-sqlite/integration_tests/integration_test.py @@ -12,6 +12,8 @@ from unittest.mock import MagicMock import pytest +from destination_sqlite import DestinationSqlite + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -23,7 +25,6 @@ SyncMode, Type, ) -from destination_sqlite import DestinationSqlite @pytest.fixture(autouse=True) diff --git a/airbyte-integrations/connectors/destination-sqlite/main.py b/airbyte-integrations/connectors/destination-sqlite/main.py index 8f641827f84c6..c3aabc20b4516 100644 --- a/airbyte-integrations/connectors/destination-sqlite/main.py +++ b/airbyte-integrations/connectors/destination-sqlite/main.py @@ -7,5 +7,6 @@ from destination_sqlite import DestinationSqlite + if __name__ == "__main__": DestinationSqlite().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py index 6cffbfc158979..2fbfb3b871721 100644 --- a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py @@ -7,6 +7,8 @@ from logging import getLogger from typing import Any, Iterable, Mapping +from timeplus import Environment, Stream + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -17,7 +19,7 @@ Status, Type, ) -from timeplus import Environment, Stream + logger = getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py index e3de7dac9e711..b231cdd0f47d0 100755 --- a/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py @@ -8,6 +8,8 @@ from typing import Any, Mapping import pytest +from destination_timeplus import DestinationTimeplus + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,7 +21,6 @@ SyncMode, Type, ) -from destination_timeplus import DestinationTimeplus @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-timeplus/main.py b/airbyte-integrations/connectors/destination-timeplus/main.py index a6f1b6b49d3ce..5c1e78fefd90c 100755 --- a/airbyte-integrations/connectors/destination-timeplus/main.py +++ b/airbyte-integrations/connectors/destination-timeplus/main.py @@ -7,5 +7,6 @@ from destination_timeplus import DestinationTimeplus + if __name__ == "__main__": DestinationTimeplus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py index fbbf027e2e283..8a629a6a4a943 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -7,10 +7,11 @@ import time from typing import Any, Iterable, Mapping +from typesense import Client + from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from destination_typesense.writer import TypesenseWriter -from typesense import Client def get_client(config: Mapping[str, Any]) -> Client: diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index 54e85d5512b74..bedaaf76fd2a8 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -9,6 +9,7 @@ from typesense import Client + logger = getLogger("airbyte") @@ -35,6 +36,6 @@ def flush(self): for stream, data in self.write_buffer: grouped_by_stream[stream].append(data) - for (stream, data) in grouped_by_stream.items(): + for stream, data in grouped_by_stream.items(): self.client.collections[stream].documents.import_(data) self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py index a57b7cc59fd2a..bc0302eee8d9d 100644 --- a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from typing import Any, Dict, Mapping import pytest +from destination_typesense.destination import DestinationTypesense, get_client +from typesense import Client + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_typesense.destination import DestinationTypesense, get_client -from typesense import Client @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-typesense/main.py b/airbyte-integrations/connectors/destination-typesense/main.py index f702d9f4c0b95..518595701e80b 100644 --- a/airbyte-integrations/connectors/destination-typesense/main.py +++ b/airbyte-integrations/connectors/destination-typesense/main.py @@ -7,5 +7,6 @@ from destination_typesense import DestinationTypesense + if __name__ == "__main__": DestinationTypesense().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py index 755d300147808..a46a518bea0fe 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py @@ -10,8 +10,10 @@ import backoff import requests + from destination_vectara.config import VectaraConfig + METADATA_STREAM_FIELD = "_ab_stream" @@ -25,7 +27,6 @@ def user_error(e: Exception) -> bool: class VectaraClient: - BASE_URL = "https://api.vectara.io/v1" def __init__(self, config: VectaraConfig): @@ -99,7 +100,6 @@ def _get_jwt_token(self): @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) def _request(self, endpoint: str, http_method: str = "POST", params: Mapping[str, Any] = None, data: Mapping[str, Any] = None): - url = f"{self.BASE_URL}/{endpoint}" current_ts = datetime.datetime.now().timestamp() diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py index 86ca2dba16f57..94cb2b16c4187 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py @@ -4,9 +4,10 @@ from typing import List, Optional -from airbyte_cdk.utils.spec_schema_transformations import resolve_refs from pydantic import BaseModel, Field +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs + class OAuth2(BaseModel): client_id: str = Field(..., title="OAuth Client ID", description="OAuth2.0 client id", order=0) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py index b324865d36ba6..3c60ed65b96cf 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py @@ -24,7 +24,6 @@ class DestinationVectara(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py index 401d279294f03..9c681b8896eed 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py @@ -6,16 +6,17 @@ from typing import Any, Dict, List, Mapping, Optional import dpath.util + from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType from destination_vectara.client import VectaraClient + METADATA_STREAM_FIELD = "_ab_stream" class VectaraWriter: - write_buffer: List[Mapping[str, Any]] = [] flush_interval = 1000 diff --git a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py index 052006303d859..ba9d1aff89e91 100644 --- a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py @@ -7,6 +7,9 @@ import unittest from typing import Any, Dict +from destination_vectara.client import VectaraClient +from destination_vectara.destination import DestinationVectara + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -19,8 +22,6 @@ SyncMode, Type, ) -from destination_vectara.client import VectaraClient -from destination_vectara.destination import DestinationVectara class VectaraIntegrationTest(unittest.TestCase): @@ -45,6 +46,7 @@ def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) ) + def _clean(self): self._client.delete_doc_by_metadata(metadata_field_name="_ab_stream", metadata_field_values=["None_mystream"]) diff --git a/airbyte-integrations/connectors/destination-vectara/main.py b/airbyte-integrations/connectors/destination-vectara/main.py index 289b411fb3181..81502c9cd22b2 100644 --- a/airbyte-integrations/connectors/destination-vectara/main.py +++ b/airbyte-integrations/connectors/destination-vectara/main.py @@ -7,5 +7,6 @@ from destination_vectara import DestinationVectara + if __name__ == "__main__": DestinationVectara().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py index c4708d59ffc9a..92d3171a229e5 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py @@ -4,6 +4,8 @@ from typing import List, Literal, Union +from pydantic import BaseModel, Field + from airbyte_cdk.destinations.vector_db_based.config import ( AzureOpenAIEmbeddingConfigModel, CohereEmbeddingConfigModel, @@ -14,7 +16,6 @@ VectorDBConfigModel, ) from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic import BaseModel, Field class UsernamePasswordAuth(BaseModel): diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py index 93adb9d825a4f..6649977530770 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py @@ -12,6 +12,7 @@ from typing import Optional import weaviate + from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD from airbyte_cdk.destinations.vector_db_based.indexer import Indexer from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py index e76a8afe4457e..d6d3233f6ba2b 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py @@ -8,14 +8,16 @@ import docker import weaviate -from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE -from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest -from airbyte_cdk.models import DestinationSyncMode, Status from destination_weaviate.destination import DestinationWeaviate from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Weaviate from pytest_docker.plugin import get_docker_ip +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status + + WEAVIATE_CONTAINER_NAME = "weaviate-test-container-will-get-deleted" diff --git a/airbyte-integrations/connectors/destination-weaviate/main.py b/airbyte-integrations/connectors/destination-weaviate/main.py index 83b1692b9225c..2b3cc1b284563 100644 --- a/airbyte-integrations/connectors/destination-weaviate/main.py +++ b/airbyte-integrations/connectors/destination-weaviate/main.py @@ -7,5 +7,6 @@ from destination_weaviate import DestinationWeaviate + if __name__ == "__main__": DestinationWeaviate().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py index 6d16b865f31c8..474bef7e3e861 100644 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py @@ -6,10 +6,11 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte_cdk.models import ConnectorSpecification, Status from destination_weaviate.config import ConfigModel from destination_weaviate.destination import DestinationWeaviate +from airbyte_cdk.models import ConnectorSpecification, Status + class TestDestinationWeaviate(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py index a5b2526e392cb..270a78608bae9 100644 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py @@ -6,11 +6,12 @@ from collections import defaultdict from unittest.mock import ANY, Mock, call, patch -from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode from destination_weaviate.config import NoAuth, TokenAuth, WeaviateIndexingConfigModel from destination_weaviate.indexer import WeaviateIndexer, WeaviatePartialBatchError +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode + class TestWeaviateIndexer(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py index 461110bfa284c..688f35cc1906e 100644 --- a/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py +++ b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py @@ -5,11 +5,13 @@ import logging from typing import Any, Iterable, Mapping -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type from xata.client import XataClient from xata.helpers import BulkProcessor +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type + + __version__ = "0.0.1" logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py index b98d151d31d35..326ff59b8e94d 100644 --- a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py @@ -7,6 +7,9 @@ from unittest.mock import Mock import pytest +from destination_xata import DestinationXata +from xata.client import XataClient + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -18,8 +21,6 @@ SyncMode, Type, ) -from destination_xata import DestinationXata -from xata.client import XataClient @pytest.fixture(name="config") diff --git a/airbyte-integrations/connectors/destination-xata/main.py b/airbyte-integrations/connectors/destination-xata/main.py index 76e7d8f087c0d..d195eab473216 100644 --- a/airbyte-integrations/connectors/destination-xata/main.py +++ b/airbyte-integrations/connectors/destination-xata/main.py @@ -7,5 +7,6 @@ from destination_xata import DestinationXata + if __name__ == "__main__": DestinationXata().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-activecampaign/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-adjust/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-adjust/main.py b/airbyte-integrations/connectors/source-adjust/main.py index a361a2be887c3..6dc1a06978ec9 100644 --- a/airbyte-integrations/connectors/source-adjust/main.py +++ b/airbyte-integrations/connectors/source-adjust/main.py @@ -4,5 +4,6 @@ from source_adjust.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/components.py b/airbyte-integrations/connectors/source-adjust/source_adjust/components.py index 141b701274e99..1cb725a98c71a 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/components.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/components.py @@ -11,7 +11,6 @@ @dataclass class AdjustSchemaLoader(JsonFileSchemaLoader): - config: Mapping[str, Any] def get_json_schema(self) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/model.py b/airbyte-integrations/connectors/source-adjust/source_adjust/model.py index 849f8cb5dc3f7..cb47761929be0 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/model.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/model.py @@ -15,6 +15,7 @@ import pydantic + BASE_METRICS = typing.Literal[ "network_cost", "network_cost_diff", diff --git a/airbyte-integrations/connectors/source-adjust/source_adjust/source.py b/airbyte-integrations/connectors/source-adjust/source_adjust/source.py index 01ec2350059d3..a07ed0ad520af 100644 --- a/airbyte-integrations/connectors/source-adjust/source_adjust/source.py +++ b/airbyte-integrations/connectors/source-adjust/source_adjust/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py index 1930c55800b8a..91a0d2ce69114 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py @@ -12,7 +12,7 @@ def config_pass(): "metrics": ["installs", "network_installs", "network_cost", "network_ecpi"], "dimensions": ["app", "partner_name", "campaign", "campaign_id_network", "campaign_network"], "additional_metrics": [], - "until_today": True + "until_today": True, } @@ -31,11 +31,7 @@ def mock_report_response(): return { "rows": [ { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,14 +40,9 @@ def mock_report_response(): "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } ], - "totals": { - "installs": 10.0, - "network_installs": 0.0, - "network_cost": 0.0, - "network_ecpi": 0.0 - }, + "totals": {"installs": 10.0, "network_installs": 0.0, "network_cost": 0.0, "network_ecpi": 0.0}, "warnings": [], } diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py index 909d26ac70ee4..4d3dc022aaa41 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/test_incremental_streams.py @@ -5,9 +5,10 @@ from datetime import datetime, timedelta from typing import Any, Mapping +from source_adjust.source import SourceAdjust + from airbyte_cdk.sources.streams import Stream from airbyte_protocol.models import SyncMode -from source_adjust.source import SourceAdjust def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py index 081ab6d9c05b5..cd7a13745507e 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py @@ -5,11 +5,12 @@ from datetime import datetime from typing import Any, Mapping -from airbyte_cdk.sources.streams import Stream -from airbyte_protocol.models import SyncMode from jsonref import requests from source_adjust.source import SourceAdjust +from airbyte_cdk.sources.streams import Stream +from airbyte_protocol.models import SyncMode + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceAdjust() @@ -31,11 +32,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp requests_mock.get(url=report_url, status_code=200, json=mock_report_response) stream = get_stream_by_name("AdjustReport", config_pass) expected_parsed_record = { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,7 +41,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } records = [] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-airtable/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-airtable/main.py b/airbyte-integrations/connectors/source-airtable/main.py index 170d6caf75b11..61d016c6e10fd 100644 --- a/airbyte-integrations/connectors/source-airtable/main.py +++ b/airbyte-integrations/connectors/source-airtable/main.py @@ -4,5 +4,6 @@ from source_airtable.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py index 3cf295b205621..a4283a1212403 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_backoff_strategy.py @@ -4,6 +4,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py index 7cde021f95525..336e8be7b7eb4 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_handler.py @@ -7,6 +7,7 @@ from typing import Mapping, Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py index 7ae780d39d15b..0a29d79dfcddd 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/airtable_error_mapping.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + AIRTABLE_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 403: ErrorResolution( response_action=ResponseAction.FAIL, diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py index 692e09ecae400..f1517a4567853 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py @@ -5,6 +5,7 @@ from typing import Any, Mapping, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py index 5ec6f022df9fa..f70fbacb499ea 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py @@ -9,11 +9,11 @@ from airbyte_cdk.models import AirbyteStream from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode + logger: logging.Logger = logging.getLogger("airbyte") class SchemaTypes: - string: Dict = {"type": ["null", "string"]} number: Dict = {"type": ["null", "number"]} diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py index 655959c751c02..8d841efd64f9b 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py @@ -18,7 +18,6 @@ class SourceAirtable(AbstractSource): - logger: logging.Logger = logging.getLogger("airbyte") streams_catalog: Iterable[Mapping[str, Any]] = [] _auth: AirtableAuth = None diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py index 39e985b948fa2..5a1ddeb608452 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpClient, HttpStream from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator @@ -15,6 +16,7 @@ from source_airtable.airtable_error_handler import AirtableErrorHandler from source_airtable.schema_helpers import SchemaHelpers + URL_BASE: str = "https://api.airtable.com/v0/" @@ -97,7 +99,6 @@ def path(self, **kwargs) -> str: class AirtableStream(HttpStream, ABC): def __init__(self, stream_path: str, stream_name: str, stream_schema, table_name: str, **kwargs): - self.stream_name = stream_name self.stream_path = stream_path self.stream_schema = stream_schema diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py index f2a42ae96efb7..161a0529852b1 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py @@ -4,10 +4,11 @@ import pytest +from source_airtable.streams import AirtableStream + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_airtable.streams import AirtableStream @pytest.fixture diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py index e9ee73055239b..3e0ae06c53b82 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py @@ -10,13 +10,7 @@ from source_airtable.airtable_backoff_strategy import AirtableBackoffStrategy -@pytest.mark.parametrize( - "response_code, expected_backoff_time", - [ - (429, 30), - (404, None) - ] -) +@pytest.mark.parametrize("response_code, expected_backoff_time", [(429, 30), (404, None)]) def test_backoff_time(response_code, expected_backoff_time): mocked_logger = MagicMock(spec=logging.Logger) backoff = AirtableBackoffStrategy(logger=mocked_logger) diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py index 67daf8eb31296..f10fe1cfe9519 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py @@ -6,21 +6,30 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from requests import Response from source_airtable.airtable_error_handler import AirtableErrorHandler from source_airtable.airtable_error_mapping import AIRTABLE_ERROR_MAPPING from source_airtable.auth import AirtableOAuth +from airbyte_cdk.sources.streams.http.error_handlers.response_models import FailureType, ResponseAction +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.mark.parametrize( "auth, json_response, error_message", [ - (TokenAuthenticator, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable"), - (AirtableOAuth, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Access Token does not have required permissions, please reauthenticate."), - (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable.") - ] + ( + TokenAuthenticator, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable", + ), + ( + AirtableOAuth, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Access Token does not have required permissions, please reauthenticate.", + ), + (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable."), + ], ) def test_interpret_response_handles_403_error(auth, json_response, error_message): mocked_authenticator = MagicMock(spec=auth) @@ -35,6 +44,7 @@ def test_interpret_response_handles_403_error(auth, json_response, error_message assert error_resolution.failure_type == FailureType.config_error assert error_resolution.error_message == error_message + def test_interpret_response_defers_to_airtable_error_mapping_for_other_errors(): mocked_logger = MagicMock(spec=logging.Logger) mocked_response = MagicMock(spec=Response) diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py index fbc0c6e9e49f2..766779eeddab6 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py @@ -4,9 +4,11 @@ import pytest +from source_airtable.auth import AirtableAuth, AirtableOAuth + from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils import AirbyteTracedException -from source_airtable.auth import AirtableAuth, AirtableOAuth + CONFIG_OAUTH = {"credentials": {"auth_method": "oauth2.0", "client_id": "sample_client_id", "client_secret": "sample_client_secret"}} diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py index d1591ab7fbb07..eab9bed768a2d 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py @@ -7,9 +7,10 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import AirbyteCatalog from source_airtable.source import SourceAirtable +from airbyte_cdk.models import AirbyteCatalog + @pytest.mark.parametrize( "status, check_passed", diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py index e9a8d96dd37ae..c31d0982e866f 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py @@ -11,7 +11,6 @@ class TestBases: - bases_instance = AirtableBases(authenticator=MagicMock()) def test_url_base(self): @@ -50,7 +49,6 @@ def test_parse_response(self, fake_bases_response, expected_bases_response, requ class TestTables: - tables_instance = AirtableTables(base_id="test_base_id", authenticator=MagicMock()) def test_path(self): diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/main.py b/airbyte-integrations/connectors/source-alpha-vantage/main.py index dcccfe7a535c2..48d5bd87d7f37 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/main.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/main.py @@ -4,5 +4,6 @@ from source_alpha_vantage.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py index fa00fb61a53dc..91d0018826b2e 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -7,6 +7,7 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py index fe5a71ac01fb7..e865183827a21 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_alpha_vantage import SourceAlphaVantage +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAlphaVantage() diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py index e8cede80087df..7cec796ec2070 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index 30a0b6957860f..c85eb27fa1067 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -4,5 +4,6 @@ from source_amazon_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py index 33233bb6448d5..e5bebdddd0ba5 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py index fd69c06fe9d18..6782766e81c65 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/run.py @@ -8,10 +8,11 @@ import traceback from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_amazon_ads import SourceAmazonAds from source_amazon_ads.config_migrations import MigrateStartDate diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 5b714f9f41eed..8d41e57a5eeae 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk import TState from airbyte_cdk.models import AdvancedAuth, AuthFlowType, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource @@ -16,6 +17,7 @@ from .spec import SourceAmazonAdsSpec from .streams import Profiles, SponsoredBrandsV3ReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream + # Oauth 2.0 authentication URL for amazon TOKEN_URL = "https://api.amazon.com/auth/o2/token" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py index 3156ac42e5a43..555e17e9c5841 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.py @@ -5,9 +5,10 @@ from enum import Enum from typing import List, Optional -from airbyte_cdk.sources.config import BaseConfig from pydantic.v1 import Field +from airbyte_cdk.sources.config import BaseConfig + class StateFilterEnum(str, Enum): enabled = "enabled" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 1c9aa88ff1210..3ed493d99d670 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -7,13 +7,15 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import requests +from pydantic.v1 import BaseModel, ValidationError + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction -from pydantic.v1 import BaseModel, ValidationError from source_amazon_ads.constants import URL_MAPPING + """ This class hierarchy may seem overcomplicated so here is a visualization of class to provide explanation why it had been done in this way. diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py index f1896621bbcae..a8917baf8c126 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py @@ -5,6 +5,7 @@ from typing import Any, Iterable, List, Mapping import requests + from airbyte_cdk.models import SyncMode from source_amazon_ads.streams.common import AmazonAdsStream diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py index 8b28b64b46b8c..8887c3ccaf6fe 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py @@ -6,6 +6,7 @@ from .products_report import SponsoredProductsReportStream + METRICS_MAP_V3 = { "purchasedAsin": [ "campaignBudgetCurrencyCode", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py index fbe3742bc7074..428958c58a3d1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py @@ -6,10 +6,12 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from source_amazon_ads.streams.report_streams.report_stream_models import ReportInfo from source_amazon_ads.streams.report_streams.report_streams import ReportStream + METRICS_MAP_V3 = { "campaigns": [ "addToCart", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index e616d58573a9a..c8a532c0d0fa3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -7,10 +7,12 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from .report_streams import ReportInfo, ReportStream + METRICS_MAP = { "campaigns": [ "campaignName", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index d3369fcfbf032..1f87d37ffb8a3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -15,11 +15,12 @@ import backoff import pendulum import requests +from pendulum import Date + from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from pendulum import Date from source_amazon_ads.streams.common import BasicAmazonAdsStream from source_amazon_ads.utils import get_typed_env, iterate_one_by_one diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py index 62444e6984e0d..5cde2adfd5218 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py @@ -6,6 +6,7 @@ import os from typing import Union + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py index b17aeca0e4e4d..d448a1ad9681a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/ad_requests/attribution_report_request_builder.py @@ -5,6 +5,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional + BRAND_REFERRAL_BONUS = "brb_bonus_amount" METRICS_MAP = { diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py index a9a989b8d8c7a..2acee74d842ce 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/config.py @@ -5,6 +5,7 @@ from airbyte_cdk.test.mock_http.response_builder import find_template + CLIENT_ID = "amzn.app-oa2-client.test" CLIENT_SECRET = "test-secret" REGION = "NA" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py index d97869cd5d9b1..4bee406a893a8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_attribution_report_streams.py @@ -6,6 +6,7 @@ from zoneinfo import ZoneInfo import freezegun + from airbyte_cdk.models import Level as LogLevel from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker @@ -17,6 +18,7 @@ from .config import ConfigBuilder from .utils import get_log_messages_by_log_level, read_stream + REPORTING_PERIOD = 90 _NOW = datetime.now(timezone.utc) _A_START_DATE = _NOW - timedelta(days=REPORTING_PERIOD) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py index 660623df5daf4..fe0d429ada68f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_report_streams.py @@ -6,11 +6,11 @@ import pendulum import requests_mock -from airbyte_cdk.models import AirbyteStateBlob +from source_amazon_ads.streams.report_streams import brands_report, display_report, products_report + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker, HttpRequestMatcher -from source_amazon_ads.streams.report_streams import brands_report, display_report, products_report from .ad_requests import ( OAuthRequestBuilder, diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py index 16f8849fdcfac..96874e60649ba 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/test_sponsored_streams.py @@ -25,6 +25,7 @@ from .config import ConfigBuilder from .utils import get_log_messages_by_log_level, read_stream + _DEFAULT_REQUEST_BODY = json.dumps({"maxResults": 100}) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py index c81c14347aa17..3e91740f9f3f2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/integrations/utils.py @@ -3,12 +3,12 @@ import operator from typing import Any, Dict, List, Optional -from airbyte_cdk.models import AirbyteMessage +from source_amazon_ads import SourceAmazonAds + +from airbyte_cdk.models import AirbyteMessage, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from source_amazon_ads import SourceAmazonAds def read_stream( diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index b810621b9eff6..831e3e53303ba 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -13,8 +13,6 @@ import pytest import requests_mock import responses -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode from freezegun import freeze_time from pendulum import Date from pytest import raises @@ -23,8 +21,12 @@ from source_amazon_ads.streams.report_streams.report_stream_models import RecordType from source_amazon_ads.streams.report_streams.report_streams import ReportGenerationFailure, ReportGenerationInProgress, TooManyRequests +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode + from .utils import read_incremental + """ METRIC_RESPONSE is gzip compressed binary representing this string: [ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index eb6cbb4dd93fd..a993909cc4557 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -6,12 +6,13 @@ from unittest import mock from urllib.parse import urlparse, urlunparse +from source_amazon_ads.config_migrations import MigrateStartDate + from airbyte_cdk.models import SyncMode from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config -from source_amazon_ads.config_migrations import MigrateStartDate def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index ee7f33aa3ce57..e9aadc718c43a 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -4,5 +4,6 @@ from source_amazon_seller_partner.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 4bdaab19df35c..d062e1d850466 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -6,6 +6,7 @@ from typing import Any, Mapping import pendulum + from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py index 39a7055af1b7a..b98903bd9863b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceAmazonSellerPartner + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py index 4b0e1e99bbe50..618a808ba0c32 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py @@ -31,6 +31,7 @@ Australia A39IBJ37TRP1C6 AU Japan A1VC38T7YXB528 JP """ + from enum import Enum from typing import Dict, Tuple diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index d2e1fda1a0c0f..623e103127c73 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -7,12 +7,13 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from requests import HTTPError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment from airbyte_protocol.models import ConnectorSpecification -from requests import HTTPError from source_amazon_seller_partner.auth import AWSAuthenticator from source_amazon_seller_partner.constants import get_marketplaces from source_amazon_seller_partner.streams import ( @@ -73,6 +74,7 @@ ) from source_amazon_seller_partner.utils import AmazonConfigException + # given the retention period: 730 DEFAULT_RETENTION_PERIOD_IN_DAYS = 730 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index b65b7eb9a845d..a414590399a0f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -17,6 +17,7 @@ import pendulum import requests import xmltodict + from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin, package_name_from_class @@ -28,6 +29,7 @@ from airbyte_protocol.models import FailureType from source_amazon_seller_partner.utils import STREAM_THRESHOLD_PERIOD, threshold_period_decorator + REPORTS_API_VERSION = "2021-06-30" ORDERS_API_VERSION = "v0" VENDORS_API_VERSION = "v1" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py index 6c340ecfe5664..7696b235dae67 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py @@ -7,6 +7,7 @@ from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType + LOG_LEVEL = logging.getLevelName("INFO") LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py index 12579417383da..6165eea8d437c 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py @@ -8,6 +8,7 @@ import pytest + os.environ["DEPLOYMENT_MODE"] = "testing" @@ -25,10 +26,7 @@ def init_kwargs() -> Dict[str, Any]: @pytest.fixture def report_init_kwargs(init_kwargs) -> Dict[str, Any]: - return { - "stream_name": "GET_TEST_REPORT", - **init_kwargs - } + return {"stream_name": "GET_TEST_REPORT", **init_kwargs} @pytest.fixture diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py index 75cccd73e3807..05c8b5c47d346 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + ACCESS_TOKEN = "test_access_token" LWA_APP_ID = "amazon_app_id" LWA_CLIENT_SECRET = "amazon_client_secret" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py index 79b1528ed0380..7d7a549dcc53b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/pagination.py @@ -7,6 +7,7 @@ from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_TOKEN_STRING = "MDAwMDAwMDAwMQ==" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index 1225aa30b1812..1a23c7c99549e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -11,17 +11,19 @@ import freezegun import pytest import requests_mock +from source_amazon_seller_partner.streams import ReportProcessingStatus + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher from airbyte_protocol.models import AirbyteStateMessage, FailureType, SyncMode -from source_amazon_seller_partner.streams import ReportProcessingStatus from .config import CONFIG_END_DATE, CONFIG_START_DATE, MARKETPLACE_ID, NOW, VENDOR_TRAFFIC_REPORT_CONFIG_END_DATE, ConfigBuilder from .request_builder import RequestBuilder from .response_builder import build_response, response_with_status from .utils import assert_message_in_log_output, config, find_template, get_stream_by_name, mock_auth, read_output + _DOCUMENT_DOWNLOAD_URL = "https://test.com/download" _REPORT_ID = "6789087632" _REPORT_DOCUMENT_ID = "report_document_id" @@ -447,14 +449,16 @@ def test_given_http_error_500_on_create_report_when_read_then_no_records_and_err @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) @HttpMocker() def test_given_http_error_not_support_account_id_of_type_vendor_when_read_then_no_records_and_error_logged( - self, stream_name: str, data_format: str, http_mocker: HttpMocker + self, stream_name: str, data_format: str, http_mocker: HttpMocker ): mock_auth(http_mocker) response_body = { "errors": [ - {"code": "InvalidInput", - "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", - "details": ""} + { + "code": "InvalidInput", + "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", + "details": "", + } ] } http_mocker.post( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py index f3eccb1b26159..35f2a10b7ec86 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_direct_fulfillment_shipping.py @@ -8,6 +8,7 @@ import freezegun import pendulum + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -28,6 +29,7 @@ from .response_builder import response_with_status from .utils import config, mock_auth, read_output + _START_DATE = pendulum.datetime(year=2023, month=1, day=1) _END_DATE = pendulum.datetime(year=2023, month=1, day=5) _REPLICATION_START_FIELD = "createdAfter" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py index a1416a505328d..3937e7fc0873d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_vendor_orders.py @@ -8,6 +8,7 @@ import freezegun import pendulum + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -28,6 +29,7 @@ from .response_builder import response_with_status from .utils import config, mock_auth, read_output + _START_DATE = pendulum.datetime(year=2023, month=1, day=1) _END_DATE = pendulum.datetime(year=2023, month=1, day=5) _REPLICATION_START_FIELD = "changedAfter" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py index 099f99f42f396..91b7f7c511d2f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py @@ -7,13 +7,14 @@ from http import HTTPStatus from typing import Any, List, Mapping, Optional +from source_amazon_seller_partner import SourceAmazonSellerPartner + from airbyte_cdk.sources.streams import Stream from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import _get_unit_test_folder from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, Level, SyncMode -from source_amazon_seller_partner import SourceAmazonSellerPartner from .config import ACCESS_TOKEN, ConfigBuilder from .request_builder import RequestBuilder diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py index d57b88a9f907c..16b924b9c8001 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py @@ -7,9 +7,10 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from source_amazon_seller_partner.streams import AnalyticsStream, IncrementalAnalyticsStream +from airbyte_cdk.models import SyncMode + class SomeAnalyticsStream(AnalyticsStream): report_name = "GET_ANALYTICS_STREAM" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index 385699cc58440..46360c3737d60 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -8,10 +8,12 @@ import pendulum import pytest import requests +from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents, RestockInventoryReports + from airbyte_cdk.models import SyncMode from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType -from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents, RestockInventoryReports + list_financial_event_groups_data = { "payload": { @@ -211,15 +213,7 @@ def test_reports_read_records_raise_on_backoff(mocker, requests_mock, caplog): requests_mock.post( "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, ) stream = RestockInventoryReports( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py index 058dfc514d779..533c96df2e5de 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -7,11 +7,13 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions, MigrateStreamNameOption from source_amazon_seller_partner.source import SourceAmazonSellerPartner +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + CMD = "check" SOURCE: Source = SourceAmazonSellerPartner() diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index 2d50e3adcb9f5..afd6f6e3c1ebc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -4,9 +4,11 @@ import pytest -from airbyte_cdk.models import SyncMode from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports +from airbyte_cdk.models import SyncMode + + START_DATE_1 = "2022-05-25T00:00:00Z" END_DATE_1 = "2022-05-26T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index a77cf2bd0b072..1bc7f5dab48f6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -7,11 +7,13 @@ from unittest.mock import patch import pytest -from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner from source_amazon_seller_partner.streams import VendorOrders from source_amazon_seller_partner.utils import AmazonConfigException +from airbyte_cdk.sources.streams import Stream + + logger = logging.getLogger("airbyte") @@ -124,19 +126,23 @@ def test_check_connection_with_orders(requests_mock, connector_config_with_repor ( "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ - ("GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + ( + "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ {"option_name": "some_name_1", "option_value": "some_value_1"}, {"option_name": "some_name_2", "option_value": "some_value_2"}, ], - ), - ] + ), + ], ), ("SOME_OTHER_STREAM", []), ), ) def test_get_stream_report_options_list(connector_config_with_report_options, report_name, stream_name_w_options): - assert list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) == stream_name_w_options + assert ( + list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) + == stream_name_w_options + ) def test_config_report_options_validation_error_duplicated_streams(connector_config_with_report_options): @@ -199,7 +205,9 @@ def test_spec(deployment_mode, common_streams_count, monkeypatch): "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", "GET_VENDOR_TRAFFIC_REPORT", } - streams_with_report_options = SourceAmazonSellerPartner().spec( - logger - ).connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + streams_with_report_options = ( + SourceAmazonSellerPartner() + .spec(logger) + .connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + ) assert len(set(streams_with_report_options).intersection(oss_only_streams)) == common_streams_count diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index cae628254c611..8d5857ec491ef 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -8,9 +8,6 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType from source_amazon_seller_partner.streams import ( IncrementalReportsAmazonSPStream, ReportProcessingStatus, @@ -18,6 +15,10 @@ VendorDirectFulfillmentShipping, ) +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + class SomeReportStream(ReportsAmazonSPStream): report_name = "GET_TEST_REPORT" @@ -246,15 +247,7 @@ def test_given_429_when_read_records_then_raise_transient_error(self, report_ini "POST", "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, reason="Forbidden", ) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amazon-sqs/main.py b/airbyte-integrations/connectors/source-amazon-sqs/main.py index 3e218a144f8f7..3d9850f97f6d6 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/main.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/main.py @@ -4,5 +4,6 @@ from source_amazon_sqs.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index e39e0d894b216..3ba1991d471a1 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = ["airbyte-cdk", "boto3"] TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "moto[sqs, iam]"] diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py index 6472649a9b9e1..4aedc4e71d9ca 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/source.py @@ -8,6 +8,8 @@ from typing import Dict, Generator import boto3 +from botocore.exceptions import ClientError + from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ( AirbyteCatalog, @@ -20,7 +22,6 @@ Type, ) from airbyte_cdk.sources.source import Source -from botocore.exceptions import ClientError class SourceAmazonSqs(Source): diff --git a/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py index 28ead98ff99ce..511b2081c087b 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/unit_tests/unit_test.py @@ -6,14 +6,15 @@ from typing import Any, Dict, Mapping import boto3 -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Status # from airbyte_cdk.sources.source import Source from moto import mock_iam, mock_sqs from moto.core import set_initial_no_auth_action_count from source_amazon_sqs import SourceAmazonSqs +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Status + @mock_iam def create_user_with_all_permissions(): diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py index cb2230f94ead1..c0b6f7e640196 100755 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py @@ -7,9 +7,10 @@ from pathlib import Path import pytest +from source_amplitude.source import SourceAmplitude + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_amplitude.source import SourceAmplitude @pytest.fixture(scope="module") diff --git a/airbyte-integrations/connectors/source-amplitude/main.py b/airbyte-integrations/connectors/source-amplitude/main.py index 14500e9c73e63..66dbbbcd80bf1 100644 --- a/airbyte-integrations/connectors/source-amplitude/main.py +++ b/airbyte-integrations/connectors/source-amplitude/main.py @@ -4,5 +4,6 @@ from source_amplitude.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py index 5fa407664b6e3..562a5e957bb27 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/components.py @@ -12,10 +12,12 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.types import Config, Record + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py index 3649e5d9b3114..12b0d3fe1614d 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_amplitude import SourceAmplitude +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAmplitude() diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py index 6ff2160da69c9..cc98c73367b56 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py @@ -5,10 +5,12 @@ from base64 import b64encode from typing import Any, List, Mapping +from source_amplitude.streams import Events + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_amplitude.streams import Events + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py index d6f22fa668d20..40112fa5e0ff3 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py @@ -12,6 +12,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream @@ -19,6 +20,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction + LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py index 7d78a625301a9..560b7248eabe6 100644 --- a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py @@ -11,11 +11,12 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from source_amplitude.components import ActiveUsersRecordExtractor, AverageSessionLengthRecordExtractor, EventsExtractor from source_amplitude.streams import Events +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + @pytest.mark.parametrize( "custom_extractor, data, expected", @@ -141,9 +142,9 @@ def test_event_read(self, requests_mock): "error_code, expectation", [ (400, pytest.raises(AirbyteTracedException)), - (404, does_not_raise()), # does not raise because response action is IGNORE + (404, does_not_raise()), # does not raise because response action is IGNORE (504, pytest.raises(AirbyteTracedException)), - (500, does_not_raise()), # does not raise because repsonse action is RETRY + (500, does_not_raise()), # does not raise because repsonse action is RETRY ], ) def test_event_errors_read(self, mocker, requests_mock, error_code, expectation): @@ -195,7 +196,6 @@ def test_events_parse_response(self, file_name, content_is_valid, records_count) assert len(parsed_response) == records_count if content_is_valid: - # RFC3339 pattern pattern = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$" diff --git a/airbyte-integrations/connectors/source-apify-dataset/components.py b/airbyte-integrations/connectors/source-apify-dataset/components.py index 0fea713e975a0..2b1710ddd8002 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/components.py +++ b/airbyte-integrations/connectors/source-apify-dataset/components.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appsflyer/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-appsflyer/main.py b/airbyte-integrations/connectors/source-appsflyer/main.py index ebf2655b2ec44..ba11450641756 100644 --- a/airbyte-integrations/connectors/source-appsflyer/main.py +++ b/airbyte-integrations/connectors/source-appsflyer/main.py @@ -4,5 +4,6 @@ from source_appsflyer.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py index 8e77d5b78019e..90cf565ac9c5a 100644 --- a/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py +++ b/airbyte-integrations/connectors/source-appsflyer/source_appsflyer/source.py @@ -12,12 +12,13 @@ import pendulum import requests +from pendulum.tz.timezone import Timezone + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum.tz.timezone import Timezone from .fields import * diff --git a/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py index cc6dae86af52c..6251fd0352f62 100644 --- a/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-appsflyer/unit_tests/test_incremental_streams.py @@ -4,7 +4,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from pytest import fixture, raises from source_appsflyer.fields import * from source_appsflyer.source import ( @@ -22,6 +21,8 @@ UninstallEvents, ) +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-asana/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-asana/main.py b/airbyte-integrations/connectors/source-asana/main.py index 5fde4e3c72d68..ade635df308f5 100644 --- a/airbyte-integrations/connectors/source-asana/main.py +++ b/airbyte-integrations/connectors/source-asana/main.py @@ -4,5 +4,6 @@ from source_asana.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-asana/source_asana/components.py b/airbyte-integrations/connectors/source-asana/source_asana/components.py index 41595bc273bd5..528d372c222b0 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/components.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/components.py @@ -7,10 +7,11 @@ from pkgutil import get_data from typing import Any, Mapping, MutableMapping, Optional, Union +from yaml import safe_load + from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState -from yaml import safe_load @dataclass diff --git a/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py b/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py index 64c2ff3945eb2..60ae8c42cb5a3 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/config_migration.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-asana/source_asana/run.py b/airbyte-integrations/connectors/source-asana/source_asana/run.py index fb05d363e6e0e..221a9a071af14 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/run.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/run.py @@ -8,11 +8,12 @@ from datetime import datetime from typing import List -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type from orjson import orjson from source_asana import SourceAsana +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type + from .config_migration import AsanaConfigMigration diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 22b0872e587b9..4e4efe46b6387 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py index 56efb7c45d89e..4bb360ca2e2a7 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py @@ -5,18 +5,23 @@ from source_asana.config_migration import AsanaConfigMigration from source_asana.source import SourceAsana + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" def test_should_migrate(): assert AsanaConfigMigration.should_migrate({"access_token": "asdfcxz"}) is True - assert AsanaConfigMigration.should_migrate({"credentials": { "option_title": "PAT Credentials", "personal_access_token": "1206938133417909" }} -) is False + assert ( + AsanaConfigMigration.should_migrate( + {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "1206938133417909"}} + ) + is False + ) def test__modify_and_save(): user_config = {"access_token": "asdfcxz"} - expected = {"credentials": { "option_title": "PAT Credentials", "personal_access_token": "asdfcxz" }} + expected = {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "asdfcxz"}} # todo: need to make the migrate a classmethod instead of staticmethod since the missing config field will fail validation source = SourceAsana(config=user_config, catalog=None, state=None) diff --git a/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ashby/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-avni/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-avni/main.py b/airbyte-integrations/connectors/source-avni/main.py index 5ab8e86addc58..7ff1ad92a8b01 100644 --- a/airbyte-integrations/connectors/source-avni/main.py +++ b/airbyte-integrations/connectors/source-avni/main.py @@ -5,9 +5,11 @@ import sys -from airbyte_cdk.entrypoint import launch from source_avni import SourceAvni +from airbyte_cdk.entrypoint import launch + + if __name__ == "__main__": source = SourceAvni() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py index d47cbf7654f6f..782f0f2100eff 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/components.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/components.py @@ -4,6 +4,7 @@ import boto3 import requests + from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator @@ -11,7 +12,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self) -> str: - username = self._username.eval(self.config) password = self._password.eval(self.config) @@ -29,7 +29,6 @@ def auth_header(self) -> str: return "auth-token" def get_client_id(self): - url_client = "https://app.avniproject.org/idp-details" response = requests.get(url_client) response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-avni/source_avni/run.py b/airbyte-integrations/connectors/source-avni/source_avni/run.py index 636ce2134b8d2..4b1154b784ae4 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/run.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_avni import SourceAvni +from airbyte_cdk.entrypoint import launch + def run(): source = SourceAvni() diff --git a/airbyte-integrations/connectors/source-avni/source_avni/source.py b/airbyte-integrations/connectors/source-avni/source_avni/source.py index e6c65ceadb7d3..23d6f53b3e51e 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/source.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py index 49f77bed58ec9..2ac44c6aaf65a 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py @@ -7,14 +7,13 @@ from source_avni.components import CustomAuthenticator -@patch('boto3.client') +@patch("boto3.client") def test_token_property(mock_boto3_client): - mock_cognito_client = Mock() mock_boto3_client.return_value = mock_cognito_client - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") source._username = Mock() source._username.eval.return_value = "test_username" source._password = Mock() @@ -22,24 +21,18 @@ def test_token_property(mock_boto3_client): source.get_client_id = Mock() source.get_client_id.return_value = "test_client_id" - mock_cognito_client.initiate_auth.return_value = { - "AuthenticationResult": { - "IdToken": "test_id_token" - } - } + mock_cognito_client.initiate_auth.return_value = {"AuthenticationResult": {"IdToken": "test_id_token"}} token = source.token mock_boto3_client.assert_called_once_with("cognito-idp", region_name="ap-south-1") mock_cognito_client.initiate_auth.assert_called_once_with( - ClientId="test_client_id", - AuthFlow="USER_PASSWORD_AUTH", - AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} + ClientId="test_client_id", AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} ) assert token == "test_id_token" + def test_get_client_id(mocker): - - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") client_id = source.get_client_id() expected_length = 26 - assert len(client_id) == expected_length \ No newline at end of file + assert len(client_id) == expected_length diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py index c2cf082ef31fb..b99a2a62f3012 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/components.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/components.py @@ -10,6 +10,7 @@ from typing import Any, Mapping, Union import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py b/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py index 6f490c8b1b112..bb54e81479229 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py index 62b485cc53037..2f6bd0adfe419 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/conftest.py @@ -11,15 +11,17 @@ import azure import docker import pytest -from airbyte_protocol.models import ConfiguredAirbyteCatalog from azure.storage.blob import BlobServiceClient, ContainerClient from azure.storage.blob._shared.authentication import SharedKeyCredentialPolicy from fastavro import parse_schema, writer from pandas import read_csv from source_azure_blob_storage import SourceAzureBlobStorage +from airbyte_protocol.models import ConfiguredAirbyteCatalog + from .utils import get_docker_ip, load_config + logger = logging.getLogger("airbyte") JSON_TO_AVRO_TYPES = {"string": "string", "integer": "long", "number": "float", "object": "record"} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py index 29c9ffd1ef1ac..7c4b5b24f679a 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/integration_test.py @@ -4,10 +4,11 @@ from typing import Any, Mapping import pytest +from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader + from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_protocol.models import ConfiguredAirbyteCatalog -from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/main.py b/airbyte-integrations/connectors/source-azure-blob-storage/main.py index 5d60acbf692af..37897c63affe8 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/main.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/main.py @@ -5,5 +5,6 @@ from source_azure_blob_storage.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py index d66169a9d7ae6..dd9430d7ee75e 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py @@ -9,19 +9,18 @@ from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message + logger = logging.getLogger("airbyte_logger") class MigrateConfig(ABC): @classmethod @abstractmethod - def should_migrate(cls, config: Mapping[str, Any]) -> bool: - ... + def should_migrate(cls, config: Mapping[str, Any]) -> bool: ... @classmethod @abstractmethod - def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: - ... + def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: ... @classmethod def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py index ed331a9c4a2b1..51ce990fd215b 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util +from pydantic import AnyUrl, BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from pydantic import AnyUrl, BaseModel, Field class Oauth2(BaseModel): diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index a394b0adf581a..5a835782741e6 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -6,15 +6,16 @@ from typing import Iterable, List, Optional, Union import pytz -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator from azure.core.credentials import AccessToken from azure.core.exceptions import ResourceNotFoundError from azure.storage.blob import BlobServiceClient, ContainerClient from smart_open import open +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator + from .spec import SourceAzureBlobStorageSpec diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py index dbe30bfeb7fe8..0ff89dec1f18f 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py @@ -6,20 +6,19 @@ def test_custom_authenticator(requests_mock): - authenticator = AzureOauth2Authenticator( - token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", - client_id="client_id", - client_secret="client_secret", - refresh_token="refresh_token", - ) + token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + client_id="client_id", + client_secret="client_secret", + refresh_token="refresh_token", + ) token_refresh_response = { "token_type": "Bearer", "scope": "https://storage.azure.com/user_impersonation https://storage.azure.com/.default", "expires_in": 5144, "ext_expires_in": 5144, "access_token": "access_token", - "refresh_token": "refresh_token" + "refresh_token": "refresh_token", } requests_mock.post("https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", json=token_refresh_response) new_token = authenticator.get_token() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py index b41630da48bd4..160a7b2bf1904 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_config_migration.py @@ -5,10 +5,11 @@ import os from typing import Any, Mapping -from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from source_azure_blob_storage import SourceAzureBlobStorage, SourceAzureBlobStorageSpec, SourceAzureBlobStorageStreamReader from source_azure_blob_storage.config_migrations import MigrateCredentials, MigrateLegacyConfig +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor + # HELPERS def load_config(config_path: str) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py index 3cec8f051aa2f..26f3296a6f534 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py @@ -12,24 +12,26 @@ from source_azure_blob_storage.spec import SourceAzureBlobStorageSpec from source_azure_blob_storage.stream_reader import AzureOauth2Authenticator, SourceAzureBlobStorageStreamReader + logger = logging.Logger("") @pytest.mark.parametrize( "credentials, expected_credentials_type", [ - ({"auth_type": "oauth2", - "tenant_id": "tenant_id", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token" - }, AzureOauth2Authenticator), - ({ - "auth_type": "storage_account_key", - "azure_blob_storage_account_key": "key1" - }, str), + ( + { + "auth_type": "oauth2", + "tenant_id": "tenant_id", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, + AzureOauth2Authenticator, + ), + ({"auth_type": "storage_account_key", "azure_blob_storage_account_key": "key1"}, str), ], - ids=["oauth2", "storage_account_key"] + ids=["oauth2", "storage_account_key"], ) def test_stream_reader_credentials(credentials: Dict, expected_credentials_type: Union[str, AzureOauth2Authenticator]): reader = SourceAzureBlobStorageStreamReader() @@ -59,9 +61,9 @@ def test_stream_reader_files_read_and_filter_by_date(): reader.config = config with patch.object(ContainerClient, "list_blobs") as blobs: blobs.return_value = [ - BlobProperties(name='sample_file_1.csv', **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_2.csv', **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_3.csv', **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}) + BlobProperties(name="sample_file_1.csv", **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_2.csv", **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_3.csv", **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}), ] files = list(reader.get_matching_files(globs=["**"], prefix=None, logger=logger)) assert len(files) == 2 diff --git a/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-azure-table/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-azure-table/main.py b/airbyte-integrations/connectors/source-azure-table/main.py index 0831f80657669..0ccc48e59be24 100644 --- a/airbyte-integrations/connectors/source-azure-table/main.py +++ b/airbyte-integrations/connectors/source-azure-table/main.py @@ -4,5 +4,6 @@ from source_azure_table.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py index 752e8472480cf..aa1b2aaceb788 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py @@ -23,14 +23,11 @@ def test_get_table_service_client_handles_exception(mocker, reader): """ Test that get_table_service_client method handles exceptions correctly. """ - mocker.patch( - "source_azure_table.azure_table.TableServiceClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableServiceClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_service_client() - + assert "Connection error" in str(exc_info.value) @@ -58,10 +55,7 @@ def test_get_table_client_handles_exception(mocker, reader): reader.get_table_client("") assert "table name is not valid." in str(exc_info.value) - mocker.patch( - "source_azure_table.azure_table.TableClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_client("valid_table_name") @@ -74,10 +68,7 @@ def test_get_tables_return(mocker, reader, tables): """ mock_client = mocker.MagicMock() mock_client.list_tables.return_value = tables.__iter__() - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) result = reader.get_tables() result_table_names = [table.name for table in result] @@ -92,10 +83,7 @@ def test_get_tables_handles_exception(mocker, reader): """ mock_client = mocker.MagicMock() mock_client.list_tables.side_effect = Exception("Failed to list tables") - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) with pytest.raises(Exception) as exc_info: reader.get_tables() diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py index 956375acda0bf..d9a455c7def74 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_source.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import AirbyteCatalog, SyncMode from source_azure_table.streams import AzureTableStream +from airbyte_cdk.models import AirbyteCatalog, SyncMode + # Tests def test_discover(mocker, config, tables, source, logger): diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bamboo-hr/components.py b/airbyte-integrations/connectors/source-bamboo-hr/components.py index 8035c61dc25d7..0f925f3bee741 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/components.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/components.py @@ -11,7 +11,6 @@ @dataclass class CustomReportsSchemaLoader(JsonFileSchemaLoader): - config: Mapping[str, Any] parameters: InitVar[Mapping[str, Any]] = {"name": "custom_reports_stream"} diff --git a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bigcommerce/components.py b/airbyte-integrations/connectors/source-bigcommerce/components.py index d9cdab67a57d1..5a823f6da0315 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/components.py +++ b/airbyte-integrations/connectors/source-bigcommerce/components.py @@ -7,6 +7,7 @@ import dpath.util import pendulum + from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @@ -20,7 +21,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Record: - kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice} for parsed_field in self._parsed_fields: date_time = parsed_field.value.eval(config, **kwargs) diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bigquery/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-bing-ads/main.py b/airbyte-integrations/connectors/source-bing-ads/main.py index c05297b01ad80..dfb7775f30711 100644 --- a/airbyte-integrations/connectors/source-bing-ads/main.py +++ b/airbyte-integrations/connectors/source-bing-ads/main.py @@ -4,5 +4,6 @@ from source_bing_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py index abeb59c2f0038..4aed49b85a71b 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -7,13 +7,14 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.error import URLError -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream from bingads.service_client import ServiceClient from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager -from source_bing_ads.client import Client from suds import sudsobject +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from source_bing_ads.client import Client + class BingAdsBaseStream(Stream, ABC): primary_key: Optional[Union[str, List[str], List[List[str]]]] = None diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py index 159aba01bf795..85e01f86d6930 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py @@ -8,16 +8,16 @@ import pandas as pd import pendulum +from numpy import nan + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from numpy import nan from source_bing_ads.base_streams import Accounts, BingAdsBaseStream from source_bing_ads.utils import transform_bulk_datetime_format_to_rfc_3339 class BingAdsBulkStream(BingAdsBaseStream, IncrementalMixin, ABC): - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) cursor_field = "Modified Time" primary_key = "Id" diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 06c1c801917c3..2238824057836 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -14,8 +14,6 @@ import backoff import pendulum -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens, OAuthWebAuthCodeGrant from bingads.exceptions import OAuthTokenRequestException from bingads.service_client import ServiceClient @@ -25,6 +23,10 @@ from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from suds import WebFault, sudsobject +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + + FILE_TYPE = "Csv" TIMEOUT_IN_MILLISECONDS = 3_600_000 diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py index dc49ab462cb49..5640921c89da5 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import _csv import re import xml.etree.ElementTree as ET from abc import ABC, abstractmethod @@ -9,20 +10,20 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union from urllib.parse import urlparse -import _csv import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from bingads import ServiceClient from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord from bingads.v13.reporting import ReportingDownloadParameters from cached_property import cached_property +from suds import WebFault, sudsobject + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from source_bing_ads.base_streams import Accounts, BingAdsStream from source_bing_ads.utils import transform_date_format_to_rfc_3339, transform_report_hourly_datetime_format_to_rfc_3339 -from suds import WebFault, sudsobject class HourlyReportTransformerMixin: diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py index 153de00864231..219e4628f8f01 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/base_test.py @@ -5,11 +5,6 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from airbyte_cdk.models import AirbyteStateMessage, SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_cdk.test.mock_http import HttpMocker -from airbyte_cdk.test.state_builder import StateBuilder from bingads.v13.bulk import BulkServiceManager from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from client_builder import build_request, response_with_status @@ -18,6 +13,12 @@ from suds.transport.https import HttpAuthenticated from suds_response_mock import mock_http_authenticated_send +from airbyte_cdk.models import AirbyteStateMessage, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.state_builder import StateBuilder + class BaseTest(TestCase): @property diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py index 1606921d8bd09..3e16e05a03c6f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/config_builder.py @@ -4,6 +4,7 @@ from airbyte_cdk.test.mock_http.response_builder import find_template + TENNANT_ID = "common" DEVELOPER_TOKEN = "test-token" REFRESH_TOKEN = "test-refresh-token" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py index 59b3bea844eab..ed8a64e1842e8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/suds_response_mock.py @@ -3,6 +3,7 @@ from suds.transport import Reply, Request from suds.transport.https import HttpAuthenticated + SEARCH_ACCOUNTS_RESPONSE = b""" 6f0a329e-4cb4-4c79-9c08-2dfe601ba05a diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py index d97762fc0bf56..01530c1fad973 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_accounts_stream.py @@ -2,15 +2,16 @@ from typing import Any, Dict, Optional, Tuple from unittest.mock import MagicMock, patch -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_cdk.test.mock_http import HttpMocker from base_test import BaseTest from source_bing_ads.source import SourceBingAds from suds.transport.https import HttpAuthenticated from suds_response_mock import mock_http_authenticated_send +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker + class TestAccountsStream(BaseTest): stream_name = "accounts" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py index 1dd16c11461f2..534dcb61a5bd4 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestAppInstallAdLabelsStream(TestBulkStream): stream_name = "app_install_ad_labels" @@ -38,7 +39,9 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-04T12:12:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-04T12:12:12.028+00:00" + } @HttpMocker() @freeze_time("2024-02-26") # mock current time as stream data available for 30 days only @@ -46,14 +49,14 @@ def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ad_labels_state", self.stream_name) self.auth_client(http_mocker) output, service_call_mock = self.read_stream( - self.stream_name, - SyncMode.incremental, - self._config, - "app_install_ad_labels_with_state", - state + self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_state", state ) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-29T12:55:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-29T12:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py index 7524b3241110c..913bf6105f00c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestAppInstallAdsStream(TestBulkStream): stream_name = "app_install_ads" @@ -38,16 +39,24 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-03-01T12:49:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-03-01T12:49:12.028+00:00" + } @HttpMocker() @freeze_time("2023-12-29") # mock current time as stream data available for 30 days only def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ads_state", self.stream_name) self.auth_client(http_mocker) - output, service_call_mock = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-01T10:55:12.028+00:00"} + output, service_call_mock = self.read_stream( + self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state + ) + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-01T10:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py index 8c375d58bdc22..6ac874274e0f7 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_budget_stream.py @@ -1,10 +1,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from freezegun import freeze_time from test_bulk_stream import TestBulkStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestBudgetStream(TestBulkStream): stream_name = "budget" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py index ce40374eed82d..fc3bc2c44b8d0 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_hourly_reports.py @@ -2,6 +2,7 @@ from test_report_stream import TestSuiteReportStream + FIRST_STATE = {"180535609": {"TimePeriod": "2023-11-12T00:00:00+00:00"}} SECOND_STATE = {"180535609": {"TimePeriod": "2023-11-13T00:00:00+00:00"}} diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py index ae222db7dbd06..7829df1614d25 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_report_stream.py @@ -4,12 +4,13 @@ from typing import Any, Optional import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.mock_http import HttpMocker from base_test import BaseTest from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from config_builder import ConfigBuilder +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.mock_http import HttpMocker + class TestReportStream(BaseTest): start_date = "2024-01-01" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py index 9a96becee8c61..7fc12f901ef17 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -88,18 +88,18 @@ def test_bulk_stream_read_with_chunks_app_install_ad_labels(mocked_client, confi app_install_ads = AppInstallAdLabels(mocked_client, config) result = app_install_ads.read_with_chunks(path=path_to_file) assert next(result) == { - 'Ad Group': None, - 'Campaign': None, - 'Client Id': 'ClientIdGoesHere', - 'Color': None, - 'Description': None, - 'Id': '-22', - 'Label': None, - 'Modified Time': None, - 'Name': None, - 'Parent Id': '-11112', - 'Status': None, - 'Type': 'App Install Ad Label' + "Ad Group": None, + "Campaign": None, + "Client Id": "ClientIdGoesHere", + "Color": None, + "Description": None, + "Id": "-22", + "Label": None, + "Modified Time": None, + "Name": None, + "Parent Id": "-11112", + "Status": None, + "Type": "App Install Ad Label", } @@ -116,17 +116,21 @@ def test_bulk_stream_read_with_chunks_ioe_error(mocked_client, config, caplog): @pytest.mark.parametrize( "stream_state, config_start_date, expected_start_date", [ - ({"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC)), + ( + {"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, + "2020-01-01", + DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC), + ), ({"another_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", None), ({}, "2020-01-01", None), ({}, "2023-10-21", DateTime(2023, 10, 21, 0, 0, 0, tzinfo=UTC)), ], - ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"] + ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"], ) def test_bulk_stream_start_date(mocked_client, config, stream_state, config_start_date, expected_start_date): mocked_client.reports_start_date = pendulum.parse(config_start_date) if config_start_date else None stream = AppInstallAds(mocked_client, config) - assert expected_start_date == stream.get_start_date(stream_state, 'some_account_id') + assert expected_start_date == stream.get_start_date(stream_state, "some_account_id") @patch.object(source_bing_ads.source, "Client") @@ -140,18 +144,10 @@ def test_bulk_stream_stream_state(mocked_client, config): assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} # stream state saved to connection state stream.state = { - "120342748234": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "27364572345": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "732645723": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "837563864": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - } + "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "27364572345": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "732645723": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "837563864": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, } assert stream.state == { "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, @@ -165,4 +161,6 @@ def test_bulk_stream_stream_state(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_bulk_stream_custom_transform_date_rfc3339(mocked_client, config): stream = AppInstallAds(mocked_client, config) - assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339("04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field]) + assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339( + "04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py index 96b2d5777f578..597c3477f13f3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py @@ -10,12 +10,13 @@ import pytest import source_bing_ads.client -from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens from bingads.v13.bulk import BulkServiceManager from bingads.v13.reporting.exceptions import ReportingDownloadException from suds import sudsobject +from airbyte_cdk.utils import AirbyteTracedException + def test_sudsobject_todict_primitive_types(): test_arr = ["1", "test", 1, [0, 0]] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index f68acb43a302d..aad542afec40e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import _csv import copy import json import xml.etree.ElementTree as ET @@ -9,11 +10,9 @@ from unittest.mock import MagicMock, Mock, patch from urllib.parse import urlparse -import _csv import pendulum import pytest import source_bing_ads -from airbyte_cdk.models import SyncMode from bingads.service_info import SERVICE_INFO_DICT_V13 from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord, _RowValues @@ -53,6 +52,9 @@ from source_bing_ads.source import SourceBingAds from suds import WebFault +from airbyte_cdk.models import SyncMode + + TEST_CONFIG = { "developer_token": "developer_token", "client_id": "client_id", @@ -259,7 +261,10 @@ def test_report_parse_response_csv_error(caplog): fake_response = MagicMock() fake_response.report_records.__iter__ = MagicMock(side_effect=_csv.Error) list(stream_report.parse_response(fake_response)) - assert "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." in caplog.messages + assert ( + "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." + in caplog.messages + ) @patch.object(source_bing_ads.source, "Client") @@ -404,10 +409,10 @@ def test_account_performance_report_monthly_stream_slices(mocked_client, config_ with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'LastYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'ThisYear'} + {"account_id": 180519267, "customer_id": 100, "time_period": "LastYear"}, + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "LastYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, ] @@ -417,10 +422,7 @@ def test_account_performance_report_monthly_stream_slices_no_time_period(mocked_ accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) - assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100}, - {'account_id': 180278106, 'customer_id': 200} - ] + assert stream_slice == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] @pytest.mark.parametrize( @@ -451,14 +453,38 @@ def test_custom_performance_report_no_last_year_stream_slices(mocked_client, con (AccountPerformanceReportHourly, "hourly_reports/account_performance.csv", "hourly_reports/account_performance_records.json"), (AdGroupPerformanceReportHourly, "hourly_reports/ad_group_performance.csv", "hourly_reports/ad_group_performance_records.json"), (AdPerformanceReportHourly, "hourly_reports/ad_performance.csv", "hourly_reports/ad_performance_records.json"), - (CampaignImpressionPerformanceReportHourly, "hourly_reports/campaign_impression_performance.csv", "hourly_reports/campaign_impression_performance_records.json"), + ( + CampaignImpressionPerformanceReportHourly, + "hourly_reports/campaign_impression_performance.csv", + "hourly_reports/campaign_impression_performance_records.json", + ), (KeywordPerformanceReportHourly, "hourly_reports/keyword_performance.csv", "hourly_reports/keyword_performance_records.json"), - (GeographicPerformanceReportHourly, "hourly_reports/geographic_performance.csv", "hourly_reports/geographic_performance_records.json"), + ( + GeographicPerformanceReportHourly, + "hourly_reports/geographic_performance.csv", + "hourly_reports/geographic_performance_records.json", + ), (AgeGenderAudienceReportHourly, "hourly_reports/age_gender_audience.csv", "hourly_reports/age_gender_audience_records.json"), - (SearchQueryPerformanceReportHourly, "hourly_reports/search_query_performance.csv", "hourly_reports/search_query_performance_records.json"), - (UserLocationPerformanceReportHourly, "hourly_reports/user_location_performance.csv", "hourly_reports/user_location_performance_records.json"), - (AccountImpressionPerformanceReportHourly, "hourly_reports/account_impression_performance.csv", "hourly_reports/account_impression_performance_records.json"), - (AdGroupImpressionPerformanceReportHourly, "hourly_reports/ad_group_impression_performance.csv", "hourly_reports/ad_group_impression_performance_records.json"), + ( + SearchQueryPerformanceReportHourly, + "hourly_reports/search_query_performance.csv", + "hourly_reports/search_query_performance_records.json", + ), + ( + UserLocationPerformanceReportHourly, + "hourly_reports/user_location_performance.csv", + "hourly_reports/user_location_performance_records.json", + ), + ( + AccountImpressionPerformanceReportHourly, + "hourly_reports/account_impression_performance.csv", + "hourly_reports/account_impression_performance_records.json", + ), + ( + AdGroupImpressionPerformanceReportHourly, + "hourly_reports/ad_group_impression_performance.csv", + "hourly_reports/ad_group_impression_performance_records.json", + ), ], ) @patch.object(source_bing_ads.source, "Client") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index a938300eedf04..f53cac5a1080e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -6,12 +6,13 @@ import pytest import source_bing_ads -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from bingads.service_info import SERVICE_INFO_DICT_V13 from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns from source_bing_ads.source import SourceBingAds +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + @patch.object(source_bing_ads.source, "Client") def test_streams_config_based(mocked_client, config): @@ -97,7 +98,6 @@ def test_clear_reporting_object_name(): @patch.object(source_bing_ads.source, "Client") def test_campaigns_request_params(mocked_client, config): - campaigns = Campaigns(mocked_client, config) request_params = campaigns.request_params(stream_slice={"account_id": "account_id"}) @@ -110,7 +110,6 @@ def test_campaigns_request_params(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_campaigns_stream_slices(mocked_client, config): - campaigns = Campaigns(mocked_client, config) accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): @@ -123,7 +122,6 @@ def test_campaigns_stream_slices(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_adgroups_stream_slices(mocked_client, config): - adgroups = AdGroups(mocked_client, config) accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) campaigns_read_records = [iter([{"Id": 11}, {"Id": 22}]), iter([{"Id": 55}, {"Id": 66}])] @@ -140,7 +138,6 @@ def test_adgroups_stream_slices(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_ads_request_params(mocked_client, config): - ads = Ads(mocked_client, config) request_params = ads.request_params(stream_slice={"ad_group_id": "ad_group_id"}) @@ -155,7 +152,6 @@ def test_ads_request_params(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_ads_stream_slices(mocked_client, config): - ads = Ads(mocked_client, config) with patch.object( diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-braintree/main.py b/airbyte-integrations/connectors/source-braintree/main.py index d4ae7bec52239..2b5b482a3bcdd 100644 --- a/airbyte-integrations/connectors/source-braintree/main.py +++ b/airbyte-integrations/connectors/source-braintree/main.py @@ -4,5 +4,6 @@ from source_braintree.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py index 7ca4cc60abfdf..6e8e40f64845f 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/common.py @@ -6,10 +6,11 @@ from typing import Any, Dict, Optional, Type import pydantic -from airbyte_cdk.sources.utils.schema_helpers import expand_refs from pydantic import BaseModel from pydantic.typing import resolve_annotations +from airbyte_cdk.sources.utils.schema_helpers import expand_refs + class AllOptional(pydantic.main.ModelMetaclass): """ diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py index 29d30182144c8..15d9235267c99 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py @@ -6,9 +6,6 @@ from typing import List, Union import requests -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor -from airbyte_cdk.sources.declarative.types import Record -from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from braintree.attribute_getter import AttributeGetter from braintree.customer import Customer as BCustomer from braintree.discount import Discount as BDiscount @@ -18,8 +15,13 @@ from braintree.subscription import Subscription as BSubscription from braintree.transaction import Transaction as BTransaction from braintree.util.xml_util import XmlUtil + +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from source_braintree.schemas import Customer, Discount, Dispute, MerchantAccount, Plan, Subscription, Transaction + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-braze/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-braze/main.py b/airbyte-integrations/connectors/source-braze/main.py index 723116b280980..28735b4616dec 100644 --- a/airbyte-integrations/connectors/source-braze/main.py +++ b/airbyte-integrations/connectors/source-braze/main.py @@ -4,5 +4,6 @@ from source_braze.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index 43f778382f75b..eab5b4e0fd40b 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", ] diff --git a/airbyte-integrations/connectors/source-braze/source_braze/components.py b/airbyte-integrations/connectors/source-braze/source_braze/components.py index 06e003ec51d82..0771877d7c639 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/components.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/components.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-braze/source_braze/run.py b/airbyte-integrations/connectors/source-braze/source_braze/run.py index 645b7a31df248..c2b1033fb4df0 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/run.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_braze import SourceBraze +from airbyte_cdk.entrypoint import launch + def run(): source = SourceBraze() diff --git a/airbyte-integrations/connectors/source-braze/source_braze/source.py b/airbyte-integrations/connectors/source-braze/source_braze/source.py index df957f37fb67d..44e98d1eb706c 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/source.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-braze/source_braze/transformations.py b/airbyte-integrations/connectors/source-braze/source_braze/transformations.py index 60a13384e2698..bd95bdf646a59 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/transformations.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/transformations.py @@ -6,6 +6,7 @@ from typing import Optional import dpath + from airbyte_cdk.sources.declarative.transformations import AddFields from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py index d478ac686adca..76ff085244e81 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from source_braze import DatetimeIncrementalSyncComponent + from airbyte_cdk.sources.declarative.requesters import RequestOption from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType -from source_braze import DatetimeIncrementalSyncComponent def test_datetime_slicer(): diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py index 2ed02ec26eaac..00314360102a1 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from source_braze import TransformToRecordComponent +from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition + def test_string_to_dict_transformation(): """ diff --git a/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-breezometer/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-callrail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-cart/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-cart/main.py b/airbyte-integrations/connectors/source-cart/main.py index c7f69c914848c..53fc51e036f4c 100644 --- a/airbyte-integrations/connectors/source-cart/main.py +++ b/airbyte-integrations/connectors/source-cart/main.py @@ -4,5 +4,6 @@ from source_cart.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-cart/source_cart/source.py b/airbyte-integrations/connectors/source-cart/source_cart/source.py index 15903dd2210cc..1bf2c0c918d69 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/source.py +++ b/airbyte-integrations/connectors/source-cart/source_cart/source.py @@ -13,12 +13,13 @@ import pendulum import requests +from pendulum.parsing.exceptions import ParserError + from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from pendulum.parsing.exceptions import ParserError from .streams import Addresses, CustomersCart, OrderItems, OrderPayments, Orders, OrderStatuses, Products diff --git a/airbyte-integrations/connectors/source-cart/source_cart/streams.py b/airbyte-integrations/connectors/source-cart/source_cart/streams.py index f6e31507f38ae..1d5d8a95aafa5 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/streams.py +++ b/airbyte-integrations/connectors/source-cart/source_cart/streams.py @@ -9,6 +9,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator @@ -93,7 +94,6 @@ def request_params( class IncrementalCartStream(CartStream, ABC): - state_checkpoint_interval = 1000 cursor_field = "updated_at" diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-chargebee/main.py b/airbyte-integrations/connectors/source-chargebee/main.py index 351ea1590b359..00e55c1473d3d 100644 --- a/airbyte-integrations/connectors/source-chargebee/main.py +++ b/airbyte-integrations/connectors/source-chargebee/main.py @@ -4,5 +4,6 @@ from source_chargebee.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py index 0f05242ec89e8..5df32652c8c5d 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/components.py @@ -155,7 +155,6 @@ def set(self) -> None: @dataclass class IncrementalSingleSliceCursor(DeclarativeCursor): - cursor_field: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py index 40a7f2665c333..d410d2cc25907 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/run.py @@ -8,11 +8,12 @@ from datetime import datetime from typing import List -from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type from orjson import orjson from source_chargebee import SourceChargebee +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type + def _get_source(args: List[str]): catalog_path = AirbyteEntrypoint.extract_catalog(args) diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py b/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py index b3903ecfc21af..1f6e516955b3c 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/source.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py index 85f0de928865a..65fdee1495555 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "site": "ConfigBuilder default site", "site_api_key": "ConfigBuilder default site api key", "start_date": "2023-01-01T06:57:44Z", - "product_catalog": "2.0" + "product_catalog": "2.0", } def with_site(self, site: str) -> "ConfigBuilder": @@ -30,4 +30,4 @@ def with_product_catalog(self, product_catalog: str) -> "ConfigBuilder": return self def build(self) -> Dict[str, Any]: - return self._config \ No newline at end of file + return self._config diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py index 0cf9d9d5a5bcd..f5cd872ef7c05 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py @@ -8,4 +8,4 @@ class ChargebeePaginationStrategy(PaginationStrategy): @staticmethod def update(response: Dict[str, Any]) -> None: - response["next_offset"] = "[1707076198000,57873868]" \ No newline at end of file + response["next_offset"] = "[1707076198000,57873868]" diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py index 1b97d9e4d6fb1..2429e0af58557 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py @@ -9,7 +9,6 @@ class ChargebeeRequestBuilder: - @classmethod def addon_endpoint(cls, site: str, site_api_key: str) -> "ChargebeeRequestBuilder": return cls("addons", site, site_api_key) @@ -69,7 +68,7 @@ def with_include_deleted(self, include_deleted: bool) -> "ChargebeeRequestBuilde return self def with_created_at_btw(self, created_at_btw: List[int]) -> "ChargebeeRequestBuilder": - self._created_at_btw = f'{created_at_btw}' + self._created_at_btw = f"{created_at_btw}" return self def with_updated_at_btw(self, updated_at_btw: List[int]) -> "ChargebeeRequestBuilder": @@ -97,7 +96,7 @@ def with_limit(self, limit: int) -> "ChargebeeRequestBuilder": return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -117,7 +116,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( @@ -126,8 +127,8 @@ def build(self) -> HttpRequest: headers={"Authorization": f"Basic {base64.b64encode((str(self._site_api_key) + ':').encode('utf-8')).decode('utf-8')}"}, ) -class ChargebeeSubstreamRequestBuilder(ChargebeeRequestBuilder): +class ChargebeeSubstreamRequestBuilder(ChargebeeRequestBuilder): @classmethod def subscription_with_scheduled_changes_endpoint(cls, site: str, site_api_key: str) -> "ChargebeeRequestBuilder": return cls("subscriptions", site, site_api_key) @@ -141,7 +142,7 @@ def with_endpoint_path(self, endpoint_path: str) -> "ChargebeeSubstreamRequestBu return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -161,7 +162,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py index f9163b6be3a87..838fb8d82dc78 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py @@ -10,5 +10,6 @@ def a_response_with_status(status_code: int) -> HttpResponse: return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code) + def a_response_with_status_and_header(status_code: int, header: Mapping[str, str]) -> HttpResponse: - return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) \ No newline at end of file + return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py index 91ee4bd28dec1..cf14caaf96acc 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "addon" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,31 +59,27 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() source = _source(catalog=catalog, config=config, state=state) return read(source, config, catalog, state, expecting_exception) + @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -96,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -106,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -120,14 +126,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -171,9 +173,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -189,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -202,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py index 0c0d2288b502e..854e4a45a7c9f 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "coupon" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +104,14 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +122,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +131,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -186,7 +181,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -202,8 +196,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,7 +208,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py index 18c98133aea86..ffd36b12a9245 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "customer" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +128,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +137,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -183,9 +184,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -201,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -214,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py index 92323100d784b..3aace07fdf288 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "event" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +104,15 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -171,7 +170,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -187,8 +185,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -200,7 +197,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py index e0fe9d45d38a8..48756d17096ff 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "hosted_page" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -168,9 +172,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -186,8 +190,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -199,9 +202,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py index ed00c2677e244..a441f7711d1b7 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "plan" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,10 +126,7 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -175,7 +176,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py index cf8e8b67164d2..d92ec52fedda5 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "site_migration_detail" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -35,9 +37,9 @@ _NO_STATE = {} _NOW = datetime.now(timezone.utc) -''' +""" Note that this is a semi-incremental stream and tests will need to be adapated accordingly -''' +""" def _a_request() -> ChargebeeRequestBuilder: @@ -61,22 +63,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -86,7 +84,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -101,8 +98,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -110,26 +106,16 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock @HttpMocker() def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination - http_mocker.get( - _a_request().build(), - _a_response().with_record(_a_record()).with_pagination().build() - ) - http_mocker.get( - _a_request().with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() - ) + http_mocker.get(_a_request().build(), _a_response().with_record(_a_record()).with_pagination().build()) + http_mocker.get(_a_request().with_offset("[1707076198000,57873868]").build(), _a_response().with_record(_a_record()).build()) self._read(_config().with_start_date(self._start_date)) # HTTPMocker ensures call are performed - @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -179,7 +165,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - # Site Migration Detail stream is a semi-incremental stream and therefore state acts differently than typical declarative incremental implementation -- state is updated to most recent cursor value read def setUp(self) -> None: @@ -197,8 +182,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date), _NO_STATE) most_recent_state = output.most_recent_state @@ -209,7 +193,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recent_cursor_value(self, http_mocker: HttpMocker) -> None: state_cursor_value = self._start_date_in_seconds + 1 record_cursor_value = state_cursor_value + 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -219,13 +203,17 @@ def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recen output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) @HttpMocker() - def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated(self, http_mocker: HttpMocker) -> None: + def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated( + self, http_mocker: HttpMocker + ) -> None: state_cursor_value = self._start_date_in_seconds record_cursor_value = self._start_date_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -235,5 +223,7 @@ def test_given_record_returned_with_cursor_value_before_state_record_is_not_read output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) assert len(output.records) == 0 diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py index 7b980fa4fbf8a..fb2ff1776ff71 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "subscription" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +128,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +137,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -186,7 +187,6 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -202,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py index f0ad3b8c62255..cb6d46f8124c4 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -20,13 +22,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder, ChargebeeSubstreamRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "subscription_with_scheduled_changes" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -62,7 +64,7 @@ def _a_parent_record() -> RecordBuilder: find_template("subscription", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) @@ -71,31 +73,24 @@ def _a_child_record() -> RecordBuilder: find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) def _a_parent_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _a_child_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription_with_scheduled_changes", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -105,7 +100,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -122,11 +116,15 @@ def test_when_read_then_records_are_extracted(self, http_mocker: HttpMocker) -> http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -137,19 +135,29 @@ def test_given_multiple_parents_when_read_then_fetch_for_each_parent(self, http_ a_parent_id = "a_subscription_test" another_parent_id = "another_subscription_test" - http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(a_parent_id)).with_record(_a_parent_record().with_id(another_parent_id)).build() + _a_parent_response() + .with_record(_a_parent_record().with_id(a_parent_id)) + .with_record(_a_parent_record().with_id(another_parent_id)) + .build(), ) http_mocker.get( - _a_child_request().with_parent_id(a_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(a_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) http_mocker.get( - _a_child_request().with_parent_id(another_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(another_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -161,11 +169,15 @@ def test_when_read_then_primary_key_is_set(self, http_mocker: HttpMocker) -> Non http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -173,15 +185,18 @@ def test_when_read_then_primary_key_is_set(self, http_mocker: HttpMocker) -> Non @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: - parent_id = "subscription_test" http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(400), ) @@ -189,15 +204,18 @@ def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocke @HttpMocker() def test_given_http_status_404_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: - parent_id = "subscription_test" http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(404), ) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py index 8ee8f4a9d3413..24527edf0f0c1 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py @@ -5,6 +5,8 @@ from unittest import TestCase import freezegun +from source_chargebee import SourceChargebee + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -19,13 +21,13 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from source_chargebee import SourceChargebee from .config import ConfigBuilder from .pagination import ChargebeePaginationStrategy from .request_builder import ChargebeeRequestBuilder from .response_builder import a_response_with_status, a_response_with_status_and_header + _STREAM_NAME = "virtual_bank_account" _SITE = "test-site" _SITE_API_KEY = "test-api-key" @@ -57,23 +59,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -83,7 +80,6 @@ def _read( @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -98,8 +94,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8))) assert len(output.records) == 2 @@ -108,12 +103,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,14 +126,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -173,9 +173,9 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): - def setUp(self) -> None: self._now = _NOW self._now_in_seconds = int(self._now.timestamp()) @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py index 54709f56fed03..4dd6cf0592897 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import pytest -from airbyte_cdk.sources.types import Record, StreamSlice from source_chargebee.components import CustomFieldTransformation, IncrementalSingleSliceCursor +from airbyte_cdk.sources.types import Record, StreamSlice + @pytest.mark.parametrize( "record, expected_record", @@ -26,11 +27,12 @@ def test_field_transformation(record, expected_record): transformed_record = transformer.transform(record) assert transformed_record == expected_record + @pytest.mark.parametrize( "record_data, expected", [ ({"pk": 1, "name": "example", "updated_at": 1662459011}, True), - ] + ], ) def test_slicer(record_data, expected): date_time_dict = {"updated_at": 1662459010} @@ -50,24 +52,17 @@ def test_slicer(record_data, expected): @pytest.mark.parametrize( "first_record, second_record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662460000}, - True), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662440000}, - False), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2"}, - False), - ({"pk": 1, "name": "example"}, - {"pk": 2, "name": "example2", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662460000}, True), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662440000}, False), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2"}, False), + ({"pk": 1, "name": "example"}, {"pk": 2, "name": "example2", "updated_at": 1662459010}, True), + ], ) def test_is_greater_than_or_equal(first_record, second_record, expected): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") assert slicer.is_greater_than_or_equal(second_record, first_record) == expected + def test_set_initial_state(): cursor_field = "updated_at" cursor_value = 999999999 @@ -75,18 +70,19 @@ def test_set_initial_state(): slicer.set_initial_state(stream_state={cursor_field: cursor_value}) assert slicer._state[cursor_field] == cursor_value + @pytest.mark.parametrize( "record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, True), + ], ) def test_should_be_synced(record, expected): cursor_field = "updated_at" slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field=cursor_field) assert slicer.should_be_synced(record) == expected + def test_stream_slices(): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") stream_slices_instance = slicer.stream_slices() diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clazar/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clickhouse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clickup-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-close-com/main.py b/airbyte-integrations/connectors/source-close-com/main.py index f80e763159393..cbd57dadfa8ab 100644 --- a/airbyte-integrations/connectors/source-close-com/main.py +++ b/airbyte-integrations/connectors/source-close-com/main.py @@ -4,5 +4,6 @@ from source_close_com.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py b/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py index 6ce18802ab7ea..71ac5b4189ce2 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/datetime_incremental_sync.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import pendulum + from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py index 2b524cb260e61..24f4a6b5f80ad 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py @@ -10,6 +10,7 @@ from urllib.parse import parse_qsl, urlparse import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py index 4dc1b251dd97e..4f45b8c93d6f2 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source_lc.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coin-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-commcare/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commcare/main.py b/airbyte-integrations/connectors/source-commcare/main.py index edd438bde5be7..fe2d48f846044 100644 --- a/airbyte-integrations/connectors/source-commcare/main.py +++ b/airbyte-integrations/connectors/source-commcare/main.py @@ -4,5 +4,6 @@ from source_commcare.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py index a6f3a7bc3a6f5..6ee0e04d4463c 100644 --- a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py @@ -9,12 +9,13 @@ from urllib.parse import parse_qs import requests +from flatten_json import flatten + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from flatten_json import flatten # Basic full refresh stream @@ -56,7 +57,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json"} return params @@ -79,7 +79,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json", "extras": "true"} return params @@ -123,7 +122,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"format": "json"} if next_page_token: params.update(next_page_token) @@ -136,7 +134,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Case(IncrementalStream): - """ docs: https://www.commcarehq.org/a/[domain]/api/[version]/case/ """ @@ -167,7 +164,6 @@ def path( def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - # start date is what we saved for forms # if self.cursor_field in self.state else (CommcareStream.last_form_date or self.initial_date) ix = self.state[self.cursor_field] @@ -234,7 +230,6 @@ def path( def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - # if self.cursor_field in self.state else self.initial_date ix = self.state[self.cursor_field] params = { diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-commercetools/main.py b/airbyte-integrations/connectors/source-commercetools/main.py index 44dd2fb8f9527..5d986dfc5e603 100644 --- a/airbyte-integrations/connectors/source-commercetools/main.py +++ b/airbyte-integrations/connectors/source-commercetools/main.py @@ -4,5 +4,6 @@ from source_commercetools.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py index f39ab7dd6f286..403963f6d5942 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py @@ -7,9 +7,11 @@ import backoff import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py index 0d264787f978d..0b5f63721a935 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_commercetools import SourceCommercetools +from airbyte_cdk.entrypoint import launch + def run(): source = SourceCommercetools() diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py index cdcca15f5522a..b36bc81e6d504 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-configcat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-convertkit/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-convex/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-convex/main.py b/airbyte-integrations/connectors/source-convex/main.py index 751ae667fae24..c407ac73f395a 100644 --- a/airbyte-integrations/connectors/source-convex/main.py +++ b/airbyte-integrations/connectors/source-convex/main.py @@ -4,5 +4,6 @@ from source_convex.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-convex/source_convex/source.py b/airbyte-integrations/connectors/source-convex/source_convex/source.py index 664f5bf3ca169..3b12ca62656f9 100644 --- a/airbyte-integrations/connectors/source-convex/source_convex/source.py +++ b/airbyte-integrations/connectors/source-convex/source_convex/source.py @@ -8,12 +8,14 @@ from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, TypedDict, cast import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator + ConvexConfig = TypedDict( "ConvexConfig", { diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py index c1006b9f61677..a8416bcaca7e2 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py @@ -5,10 +5,11 @@ from unittest.mock import MagicMock -from airbyte_cdk.models import SyncMode from pytest import fixture from source_convex.source import ConvexStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index 17512d01cf07c..7fc5a7a0158fa 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -8,9 +8,10 @@ import pytest import requests import responses -from airbyte_cdk.models import SyncMode from source_convex.source import ConvexStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def patch_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-datascope/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-db2/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-declarative-manifest/main.py b/airbyte-integrations/connectors/source-declarative-manifest/main.py index fb1e853213d7a..a4c78be28f710 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/main.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/main.py @@ -4,5 +4,6 @@ from source_declarative_manifest.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py b/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py index 2dfd888804b8c..811567aa78fa8 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/source_declarative_manifest/run.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any, List, Mapping, Optional +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import ( AirbyteErrorTraceMessage, @@ -27,7 +29,6 @@ from airbyte_cdk.sources.declarative.concurrent_declarative_source import ConcurrentDeclarativeSource from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState -from orjson import orjson class SourceLocalYaml(YamlDeclarativeSource): diff --git a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py index e2f03e369f8db..41e6254d51f14 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py @@ -9,6 +9,7 @@ from jsonschema import ValidationError from source_declarative_manifest import run + POKEAPI_JSON_SPEC_SUBSTRING = '"required":["pokemon_name"]' SUCCESS_CHECK_SUBSTRING = '"connectionStatus":{"status":"SUCCEEDED"}' FAILED_CHECK_SUBSTRING = '"connectionStatus":{"status":"FAILED"}' @@ -16,8 +17,10 @@ @pytest.fixture(autouse=True) def setup(valid_local_manifest_yaml): - with patch('source_declarative_manifest.run._is_local_manifest_command', return_value=True): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=valid_local_manifest_yaml): + with patch("source_declarative_manifest.run._is_local_manifest_command", return_value=True): + with patch( + "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=valid_local_manifest_yaml + ): yield @@ -28,9 +31,9 @@ def test_spec_is_poke_api(capsys): def test_invalid_yaml_throws(capsys, invalid_local_manifest_yaml): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=invalid_local_manifest_yaml): - with pytest.raises(ValidationError): - run.handle_command(["spec"]) + with patch("source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=invalid_local_manifest_yaml): + with pytest.raises(ValidationError): + run.handle_command(["spec"]) def test_given_invalid_config_then_unsuccessful_check(capsys, invalid_local_config_file): diff --git a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py index 52a363a298c1d..d96beb954d501 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_remote_manifest.py @@ -3,9 +3,11 @@ # import pytest -from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from source_declarative_manifest.run import create_declarative_source, handle_command +from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource + + REMOTE_MANIFEST_SPEC_SUBSTRING = '"required":["__injected_declarative_manifest"]' diff --git a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-delighted/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dremio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-elasticsearch/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py index 3a0f562732fb4..6e0d32803f452 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-everhour/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/main.py b/airbyte-integrations/connectors/source-facebook-marketing/main.py index fc25c7149e939..b0d0f19d8d9f0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/main.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/main.py @@ -5,5 +5,6 @@ from source_facebook_marketing.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index 31bf4644013da..7b7615ea92b20 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -13,8 +13,10 @@ from facebook_business.adobjects.adaccount import AdAccount from facebook_business.api import FacebookResponse from facebook_business.exceptions import FacebookRequestError + from source_facebook_marketing.streams.common import retry_pattern + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index da5b8dc1c285a..a4b28821bce2b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index b6eb3cb30c1e2..776bdcbe2d672 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -7,6 +7,7 @@ import facebook_business import pendulum + from airbyte_cdk.models import ( AdvancedAuth, AuthFlowType, @@ -55,6 +56,7 @@ from .utils import validate_end_date, validate_start_date + logger = logging.getLogger("airbyte") UNSUPPORTED_FIELDS = {"unique_conversions", "unique_ctr", "unique_clicks"} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 79c82c2210adb..6ccf6edbf8b85 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -6,14 +6,16 @@ from enum import Enum from typing import List, Literal, Optional, Set, Union -from airbyte_cdk.sources.config import BaseConfig -from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from facebook_business.adobjects.ad import Ad from facebook_business.adobjects.adset import AdSet from facebook_business.adobjects.adsinsights import AdsInsights from facebook_business.adobjects.campaign import Campaign from pydantic.v1 import BaseModel, Field, PositiveInt, constr +from airbyte_cdk.sources.config import BaseConfig +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig + + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index e8f1038c2bb97..c6da3d8727677 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -18,10 +18,12 @@ from facebook_business.adobjects.objectparser import ObjectParser from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse from pendulum.duration import Duration + from source_facebook_marketing.streams.common import retry_pattern from ..utils import validate_start_date + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index dc01cd2284127..4fcaddcfdfcc7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -10,6 +10,7 @@ from .async_job import AsyncJob, ParentAsyncJob, update_in_batch + if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index b33b9278eeb72..8b85af62a4a36 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -6,19 +6,21 @@ from functools import cache, cached_property from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union -import airbyte_cdk.sources.utils.casing as casing import pendulum +from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError + +import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.utils import AirbyteTracedException -from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob from source_facebook_marketing.streams.async_job_manager import InsightAsyncJobManager from source_facebook_marketing.streams.common import traced_exception from .base_streams import FBMarketingIncrementalStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index ed4bc29aef340..5e7562d644e9d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -9,16 +9,18 @@ from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum +from facebook_business.adobjects.abstractobject import AbstractObject +from facebook_business.exceptions import FacebookRequestError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from facebook_business.adobjects.abstractobject import AbstractObject -from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams.common import traced_exception from .common import deep_merge + if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index c3446cdfea01f..46f150e766f74 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -10,9 +10,11 @@ import backoff import pendulum +from facebook_business.exceptions import FacebookRequestError + from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException -from facebook_business.exceptions import FacebookRequestError + # The Facebook API error codes indicating rate-limiting are listed at # https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index d33e202a637bd..0dd3b4177a187 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -8,16 +8,18 @@ import pendulum import requests -from airbyte_cdk.models import SyncMode from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount from facebook_business.adobjects.adimage import AdImage from facebook_business.adobjects.user import User from facebook_business.exceptions import FacebookRequestError + +from airbyte_cdk.models import SyncMode from source_facebook_marketing.spec import ValidAdSetStatuses, ValidAdStatuses, ValidCampaignStatuses from .base_insight_streams import AdsInsights from .base_streams import FBMarketingIncrementalStream, FBMarketingReversedIncrementalStream, FBMarketingStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py index e81c6bfd14c5e..71e35ddac0075 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py @@ -8,6 +8,7 @@ import pendulum from pendulum import Date, DateTime + logger = logging.getLogger("airbyte") # Facebook store metrics maximum of 37 months old. Any time range that diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index 7a7cbaa39b9e3..1ea8602ce6561 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -7,6 +7,7 @@ from pytest import fixture from source_facebook_marketing.api import API + FB_API_VERSION = FacebookAdsApi.API_VERSION diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py index 1e76b6403a501..51772039a52ec 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + ACCESS_TOKEN = "test_access_token" ACCOUNT_ID = "111111111111111" CLIENT_ID = "test_client_id" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py index 69b284d6d308b..16846f6d78d43 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/pagination.py @@ -9,6 +9,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "QVFIUlhOX3Rnbm5Y" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 2a290cd48cbc6..8bdbe9ddd9cca 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -11,6 +11,8 @@ import freezegun import pendulum +from source_facebook_marketing.streams.async_job import Status + from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, StreamDescriptor, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker @@ -23,7 +25,6 @@ create_response_builder, find_template, ) -from source_facebook_marketing.streams.async_job import Status from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy @@ -31,113 +32,114 @@ from .response_builder import build_response, error_reduce_amount_of_data_response, get_account_response from .utils import config, encode_request_body, read_output + _STREAM_NAME = "ads_insights_action_product_id" _CURSOR_FIELD = "date_start" _REPORT_RUN_ID = "1571860060019548" _JOB_ID = "1049937379601625" _JOB_START_FIELDS = [ - "account_currency", - "account_id", - "account_name", - "action_values", - "actions", - "ad_click_actions", - "ad_id", - "ad_impression_actions", - "ad_name", - "adset_id", - "adset_name", - "attribution_setting", - "auction_bid", - "auction_competitiveness", - "auction_max_competitor_bid", - "buying_type", - "campaign_id", - "campaign_name", - "canvas_avg_view_percent", - "canvas_avg_view_time", - "catalog_segment_actions", - "catalog_segment_value", - "catalog_segment_value_mobile_purchase_roas", - "catalog_segment_value_omni_purchase_roas", - "catalog_segment_value_website_purchase_roas", - "clicks", - "conversion_rate_ranking", - "conversion_values", - "conversions", - "converted_product_quantity", - "converted_product_value", - "cost_per_15_sec_video_view", - "cost_per_2_sec_continuous_video_view", - "cost_per_action_type", - "cost_per_ad_click", - "cost_per_conversion", - "cost_per_estimated_ad_recallers", - "cost_per_inline_link_click", - "cost_per_inline_post_engagement", - "cost_per_outbound_click", - "cost_per_thruplay", - "cost_per_unique_action_type", - "cost_per_unique_click", - "cost_per_unique_inline_link_click", - "cost_per_unique_outbound_click", - "cpc", - "cpm", - "cpp", - "created_time", - "ctr", - "date_start", - "date_stop", - "engagement_rate_ranking", - "estimated_ad_recallers", - "frequency", - "full_view_impressions", - "full_view_reach", - "impressions", - "inline_link_click_ctr", - "inline_link_clicks", - "inline_post_engagement", - "instant_experience_clicks_to_open", - "instant_experience_clicks_to_start", - "instant_experience_outbound_clicks", - "mobile_app_purchase_roas", - "objective", - "optimization_goal", - "outbound_clicks", - "outbound_clicks_ctr", - "purchase_roas", - "qualifying_question_qualify_answer_rate", - "quality_ranking", - "reach", - "social_spend", - "spend", - "unique_actions", - "unique_clicks", - "unique_ctr", - "unique_inline_link_click_ctr", - "unique_inline_link_clicks", - "unique_link_clicks_ctr", - "unique_outbound_clicks", - "unique_outbound_clicks_ctr", - "updated_time", - "video_15_sec_watched_actions", - "video_30_sec_watched_actions", - "video_avg_time_watched_actions", - "video_continuous_2_sec_watched_actions", - "video_p100_watched_actions", - "video_p25_watched_actions", - "video_p50_watched_actions", - "video_p75_watched_actions", - "video_p95_watched_actions", - "video_play_actions", - "video_play_curve_actions", - "video_play_retention_0_to_15s_actions", - "video_play_retention_20_to_60s_actions", - "video_play_retention_graph_actions", - "video_time_watched_actions", - "website_ctr", - "website_purchase_roas", - ] + "account_currency", + "account_id", + "account_name", + "action_values", + "actions", + "ad_click_actions", + "ad_id", + "ad_impression_actions", + "ad_name", + "adset_id", + "adset_name", + "attribution_setting", + "auction_bid", + "auction_competitiveness", + "auction_max_competitor_bid", + "buying_type", + "campaign_id", + "campaign_name", + "canvas_avg_view_percent", + "canvas_avg_view_time", + "catalog_segment_actions", + "catalog_segment_value", + "catalog_segment_value_mobile_purchase_roas", + "catalog_segment_value_omni_purchase_roas", + "catalog_segment_value_website_purchase_roas", + "clicks", + "conversion_rate_ranking", + "conversion_values", + "conversions", + "converted_product_quantity", + "converted_product_value", + "cost_per_15_sec_video_view", + "cost_per_2_sec_continuous_video_view", + "cost_per_action_type", + "cost_per_ad_click", + "cost_per_conversion", + "cost_per_estimated_ad_recallers", + "cost_per_inline_link_click", + "cost_per_inline_post_engagement", + "cost_per_outbound_click", + "cost_per_thruplay", + "cost_per_unique_action_type", + "cost_per_unique_click", + "cost_per_unique_inline_link_click", + "cost_per_unique_outbound_click", + "cpc", + "cpm", + "cpp", + "created_time", + "ctr", + "date_start", + "date_stop", + "engagement_rate_ranking", + "estimated_ad_recallers", + "frequency", + "full_view_impressions", + "full_view_reach", + "impressions", + "inline_link_click_ctr", + "inline_link_clicks", + "inline_post_engagement", + "instant_experience_clicks_to_open", + "instant_experience_clicks_to_start", + "instant_experience_outbound_clicks", + "mobile_app_purchase_roas", + "objective", + "optimization_goal", + "outbound_clicks", + "outbound_clicks_ctr", + "purchase_roas", + "qualifying_question_qualify_answer_rate", + "quality_ranking", + "reach", + "social_spend", + "spend", + "unique_actions", + "unique_clicks", + "unique_ctr", + "unique_inline_link_click_ctr", + "unique_inline_link_clicks", + "unique_link_clicks_ctr", + "unique_outbound_clicks", + "unique_outbound_clicks_ctr", + "updated_time", + "video_15_sec_watched_actions", + "video_30_sec_watched_actions", + "video_avg_time_watched_actions", + "video_continuous_2_sec_watched_actions", + "video_p100_watched_actions", + "video_p25_watched_actions", + "video_p50_watched_actions", + "video_p75_watched_actions", + "video_p95_watched_actions", + "video_play_actions", + "video_play_curve_actions", + "video_play_retention_0_to_15s_actions", + "video_play_retention_20_to_60s_actions", + "video_play_retention_graph_actions", + "video_time_watched_actions", + "website_ctr", + "website_purchase_roas", +] def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: @@ -145,7 +147,10 @@ def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) - def _job_start_request( - account_id: Optional[str] = ACCOUNT_ID, since: Optional[datetime] = None, until: Optional[datetime] = None, fields: Optional[List[str]] = None + account_id: Optional[str] = ACCOUNT_ID, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + fields: Optional[List[str]] = None, ) -> RequestBuilder: since = since.strftime(DATE_FORMAT) if since else START_DATE[:10] until = until.strftime(DATE_FORMAT) if until else END_DATE[:10] @@ -174,7 +179,7 @@ def _job_start_request( "PENDING_BILLING_INFO", "PENDING_REVIEW", "PREAPPROVED", - "WITH_ISSUES" + "WITH_ISSUES", ], }, ], @@ -250,7 +255,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -434,7 +439,10 @@ def test_given_status_500_reduce_amount_of_data_when_read_then_limit_reduced(sel class TestIncremental(TestCase): @staticmethod def _read( - config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False, json_schema: Optional[Dict[str, any]] = None + config_: ConfigBuilder, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: bool = False, + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: return read_output( config_builder=config_, @@ -442,7 +450,7 @@ def _read( sync_mode=SyncMode.incremental, state=state, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -467,7 +475,9 @@ def test_when_read_then_state_message_produced_and_state_match_start_interval(se ) output = self._read(config().with_account_ids([account_id]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_message == start_date.strftime(DATE_FORMAT) @@ -511,8 +521,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) expected_cursor_value = start_date.strftime(DATE_FORMAT) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_account_1 == expected_cursor_value diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index d437566f1d48c..90c13e2a6941e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -7,6 +7,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker @@ -26,6 +27,7 @@ from .response_builder import error_reduce_amount_of_data_response, get_account_response from .utils import config, read_output + _STREAM_NAME = "videos" _CURSOR_FIELD = "updated_time" _FIELDS = [ @@ -96,7 +98,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -244,7 +246,9 @@ def test_when_read_then_state_message_produced_and_state_match_latest_record(sel ) output = self._read(config().with_account_ids([account_id])) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_message == max_cursor_value @HttpMocker() @@ -276,8 +280,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2])) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_account_1 == max_cursor_value_account_id_1 assert cursor_value_from_state_account_2 == max_cursor_value_account_id_2 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py index 2686186e18407..eb45b856b124d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py @@ -6,11 +6,12 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlencode +from facebook_business.api import _top_level_param_json_encode +from source_facebook_marketing import SourceFacebookMarketing + from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import ConfiguredAirbyteStreamBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from facebook_business.api import _top_level_param_json_encode -from source_facebook_marketing import SourceFacebookMarketing from .config import ConfigBuilder @@ -34,7 +35,7 @@ def read_output( sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: Optional[bool] = False, - json_schema: Optional[Dict[str, any]] = None + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: _catalog = catalog(stream_name, sync_mode, json_schema) _config = config_builder.build() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py index c09279ca1d8dd..0bd249e56bc40 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py @@ -8,6 +8,7 @@ from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.adobjects.adaccount import AdAccount + FB_API_VERSION = FacebookAdsApi.API_VERSION diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 9b9a0e2a1f9fe..86a58ffadcfaf 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -6,12 +6,13 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from freezegun import freeze_time from pendulum import duration from source_facebook_marketing.streams import AdsInsights from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob +from airbyte_cdk.models import SyncMode + @pytest.fixture(name="api") def api_fixture(mocker): @@ -99,14 +100,10 @@ def test_init_statuses(self, api, some_config): end_date=datetime(2011, 1, 1), insights_lookback_window=28, fields=["account_id", "account_currency"], - filter_statuses=["ACTIVE", "ARCHIVED"] + filter_statuses=["ACTIVE", "ARCHIVED"], ) - assert stream.request_params()["filtering"] == [ - {'field': 'ad.effective_status', - 'operator': 'IN', - 'value': ['ACTIVE', 'ARCHIVED']} - ] + assert stream.request_params()["filtering"] == [{"field": "ad.effective_status", "operator": "IN", "value": ["ACTIVE", "ARCHIVED"]}] def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock @@ -464,7 +461,9 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) - assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1, "should be 37 slices because we ignore slices which are within insights_lookback_period" + assert ( + len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1 + ), "should be 37 slices because we ignore slices which are within insights_lookback_period" assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1) @@ -602,61 +601,78 @@ def test_start_date_with_lookback_window( @pytest.mark.parametrize( "breakdowns, record, expected_record", ( - ( - ["body_asset", ], - {"body_asset": {"id": "871246182", "text": "Some text"}}, - {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"} - ), - ( - ["call_to_action_asset",], - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"} - ), - ( - ["description_asset", ], - {"description_asset": {"id": "871246182", "text": "Some text"}}, - {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"} - ), - ( - ["image_asset", ], - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"} - ), - ( - ["link_url_asset", ], - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"} - ), - ( - ["title_asset", ], - {"title_asset": {"id": "871246182", "text": "Some text"}}, - {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"} - ), - ( - ["video_asset", ], - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - } - }, - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - }, - "video_asset_id": "871246182" - } - ), - ( - ["body_asset", "country"], - {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, - { - "body_asset": {"id": "871246182", "text": "Some text"}, - "country": "country", "dma": "dma", "body_asset_id": "871246182" - } - ), - ) + ( + [ + "body_asset", + ], + {"body_asset": {"id": "871246182", "text": "Some text"}}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"}, + ), + ( + [ + "call_to_action_asset", + ], + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"}, + ), + ( + [ + "description_asset", + ], + {"description_asset": {"id": "871246182", "text": "Some text"}}, + {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"}, + ), + ( + [ + "image_asset", + ], + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"}, + ), + ( + [ + "link_url_asset", + ], + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"}, + ), + ( + [ + "title_asset", + ], + {"title_asset": {"id": "871246182", "text": "Some text"}}, + {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"}, + ), + ( + [ + "video_asset", + ], + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + } + }, + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + }, + "video_asset_id": "871246182", + }, + ), + ( + ["body_asset", "country"], + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma", "body_asset_id": "871246182"}, + ), + ), ) def test_transform_breakdowns(self, api, some_config, breakdowns, record, expected_record): start_date = pendulum.parse("2024-01-01") @@ -674,36 +690,19 @@ def test_transform_breakdowns(self, api, some_config, breakdowns, record, expect @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - None, - ["date_start", "account_id", "ad_id"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + (None, ["date_start", "account_id", "ad_id"]), + ), ) def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -714,43 +713,38 @@ def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) assert stream.primary_key == expect_pks @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], - ["date_start", "account_id", "ad_id", "video_asset_id", "link_url_asset_id", "skan_conversion_id", "place_page_id", "gender"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + ( + ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], + [ + "date_start", + "account_id", + "ad_id", + "video_asset_id", + "link_url_asset_id", + "skan_conversion_id", + "place_page_id", + "gender", + ], + ), + ), ) def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -761,7 +755,7 @@ def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pk start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) schema = stream.get_json_schema() assert schema diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 885560c92d3a9..65b53d21eb115 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -6,12 +6,14 @@ import pendulum import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams import Activities, AdAccount, AdCreatives, Campaigns, Videos +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException + + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -105,7 +107,9 @@ def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, except FacebookRequestError: pytest.fail("Call rate error has not being handled") - def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception(self, requests_mock, api, fb_call_rate_response, account_id, some_config): + def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception( + self, requests_mock, api, fb_call_rate_response, account_id, some_config + ): requests_mock.register_uri( "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 68229d5923a1a..91a1a9b1a174e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -8,8 +8,6 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_facebook_marketing.config_migrations import ( MigrateAccountIdToArray, MigrateIncludeDeletedToStatusFilters, @@ -17,6 +15,10 @@ ) from source_facebook_marketing.source import SourceFacebookMarketing +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" SOURCE: Source = SourceFacebookMarketing() @@ -168,6 +170,7 @@ def test_should_not_migrate_upgraded_config(self): migration_instance = MigrateIncludeDeletedToStatusFilters() assert not migration_instance.should_migrate(new_config) + class TestMigrateSecretsPathInConnector: OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_access_token_config.json") NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_access_token_config.json") @@ -178,7 +181,7 @@ class TestMigrateSecretsPathInConnector: def revert_migration(config_path: str) -> None: with open(config_path, "r") as test_config: config = json.load(test_config) - credentials = config.pop("credentials",{}) + credentials = config.pop("credentials", {}) credentials.pop("auth_type", None) with open(config_path, "w") as updated_config: config = json.dumps({**config, **credentials}) @@ -202,7 +205,7 @@ def test_migrate_access_token_config(self): assert original_config["access_token"] == credentials["access_token"] # revert the test_config to the starting point self.revert_migration(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) - + def test_migrate_client_config(self): migration_instance = MigrateSecretsPathInConnector() original_config = load_config(self.OLD_TEST_CONFIG_PATH_CLIENT) @@ -228,7 +231,7 @@ def test_should_not_migrate_new_client_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) migration_instance = MigrateSecretsPathInConnector() assert not migration_instance._should_migrate(new_config) - + def test_should_not_migrate_new_access_token_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_ACCESS_TOKEN) migration_instance = MigrateSecretsPathInConnector() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py index b593e7e1b1f3c..258c32425638d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -7,14 +7,16 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.api import API from source_facebook_marketing.streams import AdAccount, AdCreatives, AdsInsights from source_facebook_marketing.streams.common import traced_exception +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + FB_API_VERSION = FacebookAdsApi.API_VERSION account_id = "unknown_account" @@ -113,7 +115,7 @@ "code": 100, } }, - } + }, # Error randomly happens for different connections. # Can be reproduced on https://developers.facebook.com/tools/explorer/?method=GET&path=act_&version=v17.0 # 1st reason: incorrect ad account id is used @@ -183,7 +185,7 @@ "error_user_msg": "profile should always be linked to delegate page", } }, - } + }, # Error happens on Video stream: https://graph.facebook.com/v17.0/act_XXXXXXXXXXXXXXXX/advideos # Recommendations says that the problem can be fixed by switching to Business Ad Account Id ), @@ -215,8 +217,8 @@ "message": "(#3018) The start date of the time range cannot be beyond 37 months from the current date", "type": "OAuthException", "code": 3018, - "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P" - } + "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P", + } }, }, ), @@ -252,27 +254,28 @@ SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME = "error_400_service_temporarily_unavailable" SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE = { - "status_code": 503, - "json": { - "error": { - "message": "(#2) Service temporarily unavailable", - "type": "OAuthException", - "is_transient": True, - "code": 2, - "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", - } - }, - } + "status_code": 503, + "json": { + "error": { + "message": "(#2) Service temporarily unavailable", + "type": "OAuthException", + "is_transient": True, + "code": 2, + "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", + } + }, +} REDUCE_FIELDS_ERROR_TEST_NAME = "error_500_reduce_the_amount_of_data" REDUCE_FIELDS_ERROR_RESPONSE = { - "status_code": 500, - "json": { - "error": { - "message": "Please reduce the amount of data you're asking for, then retry your request", - "code": 1, - } - }, - } + "status_code": 500, + "json": { + "error": { + "message": "Please reduce the amount of data you're asking for, then retry your request", + "code": 1, + } + }, +} + class TestRealErrors: @pytest.mark.parametrize( @@ -406,23 +409,24 @@ def test_config_error_during_actual_nodes_read(self, requests_mock, name, friend assert error.failure_type == FailureType.config_error assert friendly_msg in error.message - @pytest.mark.parametrize("name, friendly_msg, config_error_response, failure_type", - [ - ( - REDUCE_FIELDS_ERROR_TEST_NAME, - "Please reduce the number of fields requested. Go to the schema tab, " - "select your source, and unselect the fields you do not need.", - REDUCE_FIELDS_ERROR_RESPONSE, - FailureType.config_error - ), - ( - SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, - "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", - SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, - FailureType.transient_error - ) - ] - ) + @pytest.mark.parametrize( + "name, friendly_msg, config_error_response, failure_type", + [ + ( + REDUCE_FIELDS_ERROR_TEST_NAME, + "Please reduce the number of fields requested. Go to the schema tab, " + "select your source, and unselect the fields you do not need.", + REDUCE_FIELDS_ERROR_RESPONSE, + FailureType.config_error, + ), + ( + SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, + "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", + SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, + FailureType.transient_error, + ), + ], + ) def test_config_error_that_was_retried_when_reading_nodes(self, requests_mock, name, friendly_msg, config_error_response, failure_type): """This test covers errors that have been resolved in the past with a retry strategy, but it could also can fail after retries, then, we need to provide the user with a humanized error explaining what just happened""" @@ -498,7 +502,7 @@ def test_config_error_insights_during_actual_nodes_read(self, requests_mock, nam assert friendly_msg in error.message def test_retry_for_cannot_include_error(self, requests_mock): - """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868 """ + """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868""" api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( @@ -516,7 +520,7 @@ def test_retry_for_cannot_include_error(self, requests_mock): "error": { "message": "(#100) Cannot include video_avg_time_watched_actions, video_continuous_2_sec_watched_actions in summary param because they weren't there while creating the report run.", "type": "OAuthException", - "code": 100 + "code": 100, } }, } @@ -594,24 +598,22 @@ def test_traced_exception_with_api_error(): request_context={}, http_status=400, http_headers={}, - body='{"error": {"message": "Error validating access token", "code": 190}}' + body='{"error": {"message": "Error validating access token", "code": 190}}', ) error.api_error_message = MagicMock(return_value="Error validating access token") - + result = traced_exception(error) - + assert isinstance(result, AirbyteTracedException) - assert result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + assert ( + result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + ) assert result.failure_type == FailureType.config_error def test_traced_exception_without_api_error(): error = FacebookRequestError( - message="Call was unsuccessful. The Facebook API has imploded", - request_context={}, - http_status=408, - http_headers={}, - body='{}' + message="Call was unsuccessful. The Facebook API has imploded", request_context={}, http_status=408, http_headers={}, body="{}" ) error.api_error_message = MagicMock(return_value=None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 774f83c1c9e5e..c5a654daa7c98 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -7,6 +7,10 @@ from unittest.mock import call import pytest +from facebook_business import FacebookAdsApi, FacebookSession +from source_facebook_marketing import SourceFacebookMarketing +from source_facebook_marketing.spec import ConnectorConfig + from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -18,9 +22,6 @@ Status, SyncMode, ) -from facebook_business import FacebookAdsApi, FacebookSession -from source_facebook_marketing import SourceFacebookMarketing -from source_facebook_marketing.spec import ConnectorConfig from .utils import command_check diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py index 3a0ac0691c2c2..209870c38ccdd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py @@ -7,6 +7,7 @@ import pytest from source_facebook_marketing.utils import DATA_RETENTION_PERIOD, validate_end_date, validate_start_date + TODAY = pendulum.datetime(2023, 3, 31) diff --git a/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-facebook-pages/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-facebook-pages/main.py b/airbyte-integrations/connectors/source-facebook-pages/main.py index 466fc2800442d..ed626c5ef7091 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/main.py +++ b/airbyte-integrations/connectors/source-facebook-pages/main.py @@ -4,5 +4,6 @@ from source_facebook_pages.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py index d730dc2ba1695..07df178e89d50 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/components.py @@ -9,12 +9,13 @@ import dpath.util import pendulum import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState -from requests import HTTPError @dataclass diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py index 3b70710fe59d1..9dd391453971a 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_facebook_pages import SourceFacebookPages +from airbyte_cdk.entrypoint import launch + def run(): source = SourceFacebookPages() diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py index c7f4b7e08d910..e6b19d27cc0ca 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py b/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py index 97e14aa22c05a..ae98e528c8659 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py +++ b/airbyte-integrations/connectors/source-facebook-pages/tools/schema_gen.py @@ -107,6 +107,7 @@ import json import os + spec_path = "facebook-business-sdk-codegen/api_specs/specs" fb_node_files = os.listdir(spec_path) fb_node_files.sort() @@ -230,7 +231,6 @@ def is_node(name): def get_fields(fields, with_refs=False): - # process Node's fields schema_fields = {} for attr in fields: @@ -282,7 +282,6 @@ def get_fields(fields, with_refs=False): def get_edges(edges): - schema_edges = {} attrs = {} for attr in edges: @@ -329,7 +328,6 @@ def get_edges(edges): def build_schema(node_name, with_refs=False): - file_path = f"{spec_path}/{node_name}.json" print(f"Fetching schema from file: {file_path}") @@ -353,7 +351,6 @@ def build_schema(node_name, with_refs=False): print(f"Process main nodes: {MAIN_NODES}") for node_name in MAIN_NODES: - page_schema = build_schema(node_name=node_name, with_refs=True) SCHEMA = {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": page_schema} diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py index 36fe896ca5d84..8f553d3d745ce 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py @@ -9,9 +9,10 @@ def test_field_transformation(): - with open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, open( - f"{os.path.dirname(__file__)}/transformed_record.json", "r" - ) as transformed_record: + with ( + open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, + open(f"{os.path.dirname(__file__)}/transformed_record.json", "r") as transformed_record, + ): initial_record = json.loads(initial_record.read()) transformed_record = json.loads(transformed_record.read()) record_transformation = CustomFieldTransformation(config={}, parameters={"name": "page"}) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py index 9df2974ae7bda..27ee46fc769b7 100644 --- a/airbyte-integrations/connectors/source-faker/main.py +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -5,5 +5,6 @@ from source_faker.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py index 70f166a8a4fbd..6be3cd5510c58 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -6,9 +6,10 @@ from multiprocessing import current_process from typing import Dict, List -from airbyte_cdk.models import AirbyteRecordMessage, Type from mimesis import Datetime, Numeric +from airbyte_cdk.models import AirbyteRecordMessage, Type + from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON from .utils import format_airbyte_time, now_millis diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index 15423b7fcb81e..965d2b71b68a3 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -10,6 +10,7 @@ from .streams import Products, Purchases, Users + DEFAULT_COUNT = 1_000 diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py index 2e8a0b7b21920..4251b42ac01bd 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -5,10 +5,11 @@ import datetime from multiprocessing import current_process -from airbyte_cdk.models import AirbyteRecordMessage, Type from mimesis import Address, Datetime, Person from mimesis.locales import Locale +from airbyte_cdk.models import AirbyteRecordMessage, Type + from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON from .utils import format_airbyte_time, now_millis diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index e4fbef60201b4..395a0181e47bb 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -4,9 +4,10 @@ import jsonschema import pytest -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type from source_faker import SourceFaker +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type + class MockLogger: def debug(a, b, **kwargs): diff --git a/airbyte-integrations/connectors/source-fastbill/components.py b/airbyte-integrations/connectors/source-fastbill/components.py index 002f96dd713ca..2d3160cd6992a 100644 --- a/airbyte-integrations/connectors/source-fastbill/components.py +++ b/airbyte-integrations/connectors/source-fastbill/components.py @@ -12,7 +12,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self): - username = self._username.eval(self.config).encode("latin1") password = self._password.eval(self.config).encode("latin1") encoded_credentials = base64.b64encode(b":".join((username, password))).strip() diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fauna/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fauna/main.py b/airbyte-integrations/connectors/source-fauna/main.py index 9e4bc25307ed1..85e9d886d2392 100644 --- a/airbyte-integrations/connectors/source-fauna/main.py +++ b/airbyte-integrations/connectors/source-fauna/main.py @@ -4,5 +4,6 @@ from source_fauna.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index 25c4e60b8647d..83c3a7c2ea4d2 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "faunadb~=4.2", diff --git a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py index 8cc685a1586cc..eb1d5cdbc9f21 100644 --- a/airbyte-integrations/connectors/source-fauna/source_fauna/source.py +++ b/airbyte-integrations/connectors/source-fauna/source_fauna/source.py @@ -8,6 +8,12 @@ from datetime import datetime from typing import Dict, Generator, Optional +from faunadb import _json +from faunadb import query as q +from faunadb.client import FaunaClient +from faunadb.errors import FaunaError, Unauthorized +from faunadb.objects import Ref + from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ( AirbyteCatalog, @@ -23,11 +29,6 @@ Type, ) from airbyte_cdk.sources import Source -from faunadb import _json -from faunadb import query as q -from faunadb.client import FaunaClient -from faunadb.errors import FaunaError, Unauthorized -from faunadb.objects import Ref from source_fauna.serialize import fauna_doc_to_airbyte diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py index c6c8263cc852a..5c6453b7748de 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/check_test.py @@ -4,13 +4,14 @@ from unittest.mock import MagicMock, Mock -from airbyte_cdk.models import Status from faunadb import query as q from faunadb.errors import Unauthorized from faunadb.objects import Ref from source_fauna import SourceFauna from test_util import config, mock_logger +from airbyte_cdk.models import Status + def query_hardcoded(expr): print(expr) diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py index 5137d192adaf6..1629a6bc530ff 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py @@ -11,6 +11,10 @@ from datetime import datetime import docker +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, DeletionsConfig, FullConfig, config, mock_logger, ref + from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteStream, @@ -21,9 +25,6 @@ SyncMode, Type, ) -from faunadb import query as q -from source_fauna import SourceFauna -from test_util import CollectionConfig, DeletionsConfig, FullConfig, config, mock_logger, ref def setup_database(source: SourceFauna): diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py index 82ac8183af346..7def4507c2ff8 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/discover_test.py @@ -4,12 +4,13 @@ from unittest.mock import MagicMock, Mock -from airbyte_cdk.models import AirbyteStream from faunadb import query as q from faunadb.objects import Ref from source_fauna import SourceFauna from test_util import config, mock_logger +from airbyte_cdk.models import AirbyteStream + def mock_source() -> SourceFauna: source = SourceFauna() diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py index 9a955244a5d40..a95880aad353d 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py @@ -6,6 +6,11 @@ from typing import Dict, Generator from unittest.mock import MagicMock, Mock +from faunadb import _json +from faunadb import query as q +from source_fauna import SourceFauna +from test_util import CollectionConfig, config, expand_columns_query, mock_logger, ref + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,10 +22,7 @@ SyncMode, Type, ) -from faunadb import _json -from faunadb import query as q -from source_fauna import SourceFauna -from test_util import CollectionConfig, config, expand_columns_query, mock_logger, ref + NOW = 1234512987 diff --git a/airbyte-integrations/connectors/source-file/build_customization.py b/airbyte-integrations/connectors/source-file/build_customization.py index 1c585de9caace..83411796d1532 100644 --- a/airbyte-integrations/connectors/source-file/build_customization.py +++ b/airbyte-integrations/connectors/source-file/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py index 758c1118eae61..38adaeb45a1f6 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/client_storage_providers_test.py @@ -9,6 +9,7 @@ import pytest from source_file.client import Client + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py index cb8041a4f3967..465dcb84e2bf5 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -24,6 +24,7 @@ from paramiko.client import AutoAddPolicy, SSHClient from paramiko.ssh_exception import SSHException + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py index 3f9980d1c0bfc..69379aa6586bd 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py @@ -7,10 +7,12 @@ from pathlib import Path import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_file import SourceFile from source_file.client import Client +from airbyte_cdk.utils import AirbyteTracedException + + SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/formats") diff --git a/airbyte-integrations/connectors/source-file/main.py b/airbyte-integrations/connectors/source-file/main.py index 3e7e82fd61d88..98e8e365ff72a 100644 --- a/airbyte-integrations/connectors/source-file/main.py +++ b/airbyte-integrations/connectors/source-file/main.py @@ -4,5 +4,6 @@ from source_file.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index 2a0e021df1cb0..ca197b53d3d01 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -24,9 +24,6 @@ import pandas as pd import smart_open import smart_open.ssh -from airbyte_cdk.entrypoint import logger -from airbyte_cdk.models import AirbyteStream, FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment from azure.storage.blob import BlobServiceClient from genson import SchemaBuilder from google.cloud.storage import Client as GCSClient @@ -38,8 +35,13 @@ from urllib3.exceptions import ProtocolError from yaml import safe_load +from airbyte_cdk.entrypoint import logger +from airbyte_cdk.models import AirbyteStream, FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment + from .utils import LOCAL_STORAGE_NAME, backoff_handler + SSH_TIMEOUT = 60 # Force the log level of the smart-open logger to ERROR - https://github.com/airbytehq/airbyte/pull/27157 diff --git a/airbyte-integrations/connectors/source-file/source_file/utils.py b/airbyte-integrations/connectors/source-file/source_file/utils.py index ef8c258b91796..248ee36e9f6e7 100644 --- a/airbyte-integrations/connectors/source-file/source_file/utils.py +++ b/airbyte-integrations/connectors/source-file/source_file/utils.py @@ -5,6 +5,7 @@ import logging from urllib.parse import parse_qs, urlencode, urlparse + # default logger logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index fc28d7e66c558..326f6475326e8 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -8,13 +8,14 @@ import pandas as pd import pytest -from airbyte_cdk.utils import AirbyteTracedException from pandas import read_csv, read_excel, testing from paramiko import SSHException from source_file.client import Client, URLFile from source_file.utils import backoff_handler from urllib3.exceptions import ProtocolError +from airbyte_cdk.utils import AirbyteTracedException + @pytest.fixture def wrong_format_client(): @@ -99,8 +100,7 @@ def test_load_dataframes_xlsx(config, absolute_path, test_files, file_name, shou assert read_file.equals(expected) -@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), - ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) +@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) def test_load_nested_json(client, config, absolute_path, test_files, file_format, file_path): if file_format == "jsonl": config["format"] = file_format @@ -131,7 +131,8 @@ def test_cache_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv" with open(f, mode="rb") as file: assert client._cache_stream(file) - + + def test_unzip_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv.zip" with open(f, mode="rb") as file: @@ -223,10 +224,11 @@ def patched_open(self): def test_backoff_handler(caplog): details = {"tries": 1, "wait": 1} backoff_handler(details) - expected = [('airbyte', 20, 'Caught retryable error after 1 tries. Waiting 1 seconds then retrying...')] + expected = [("airbyte", 20, "Caught retryable error after 1 tries. Waiting 1 seconds then retrying...")] assert caplog.record_tuples == expected + def generate_excel_file(data): """ Helper function to generate an Excel file with the given data. diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py b/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py index 14bd37a419aaa..8466ad5c82f25 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_nested_json_schema.py @@ -10,6 +10,7 @@ import pytest from source_file.source import SourceFile + json_obj = { "id": "0001", "type": "donut", diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py index 0a0c0ca798ad3..26446e5db6d1e 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py @@ -8,6 +8,8 @@ import jsonschema import pytest +from source_file.source import SourceFile + from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteMessage, @@ -22,7 +24,7 @@ ) from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models.airbyte_protocol import Type as MessageType -from source_file.source import SourceFile + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py index 50bba3cdce943..b7498c7869f76 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py @@ -10,6 +10,7 @@ import docker import pytest + pytest_plugins = ("source_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py index 708648fa8c153..58586ddc6928a 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py @@ -4,5 +4,6 @@ from source_firebase_realtime_database.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py index a1370a44d2897..4de83ba31dda6 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py @@ -35,7 +35,6 @@ ], ) def test_stream_name_from(config, stream_name): - actual = SourceFirebaseRealtimeDatabase.stream_name_from(config) expected = stream_name diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py index c66074e0551b8..d6c185fd5de8b 100644 --- a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py @@ -8,6 +8,10 @@ from typing import Dict, Generator from unittest.mock import MagicMock +from firebolt.db import Connection +from pytest import fixture +from source_firebolt.source import SourceFirebolt, establish_connection + from airbyte_cdk.models import Status from airbyte_cdk.models.airbyte_protocol import ( AirbyteStream, @@ -16,9 +20,6 @@ DestinationSyncMode, SyncMode, ) -from firebolt.db import Connection -from pytest import fixture -from source_firebolt.source import SourceFirebolt, establish_connection @fixture(scope="module") diff --git a/airbyte-integrations/connectors/source-firebolt/main.py b/airbyte-integrations/connectors/source-firebolt/main.py index a901e9c4ae299..3aa31e6e1953b 100644 --- a/airbyte-integrations/connectors/source-firebolt/main.py +++ b/airbyte-integrations/connectors/source-firebolt/main.py @@ -4,5 +4,6 @@ from source_firebolt.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py index 2d53f9c906633..aaf4e81126439 100644 --- a/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py @@ -20,6 +20,7 @@ from .database import establish_connection, get_table_structure from .utils import airbyte_message_from_data, convert_type + SUPPORTED_SYNC_MODES = [SyncMode.full_refresh] diff --git a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py index 2e273232b3237..01ab8d4942021 100644 --- a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py +++ b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py @@ -6,6 +6,11 @@ from decimal import Decimal from unittest.mock import MagicMock, patch +from pytest import fixture, mark +from source_firebolt.database import get_table_structure, parse_config +from source_firebolt.source import SUPPORTED_SYNC_MODES, SourceFirebolt, convert_type, establish_connection +from source_firebolt.utils import airbyte_message_from_data, format_fetch_result + from airbyte_cdk.models import ( AirbyteMessage, AirbyteRecordMessage, @@ -17,10 +22,6 @@ SyncMode, Type, ) -from pytest import fixture, mark -from source_firebolt.database import get_table_structure, parse_config -from source_firebolt.source import SUPPORTED_SYNC_MODES, SourceFirebolt, convert_type, establish_connection -from source_firebolt.utils import airbyte_message_from_data, format_fetch_result @fixture(params=["my_engine", "my_engine.api.firebolt.io"]) @@ -33,6 +34,7 @@ def config(request): } return args + @fixture() def legacy_config(request): args = { @@ -44,6 +46,7 @@ def legacy_config(request): } return args + @fixture() def config_no_engine(): args = { @@ -109,12 +112,14 @@ def test_parse_config(config, logger): result = parse_config(config, logger) assert result["engine_url"] == "override_engine.api.firebolt.io" + def test_parse_legacy_config(legacy_config, logger): result = parse_config(legacy_config, logger) assert result["database"] == "my_database" assert result["auth"].username == "my@username" assert result["auth"].password == "my_password" + @patch("source_firebolt.database.connect") def test_connection(mock_connection, config, config_no_engine, logger): establish_connection(config, logger) diff --git a/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fleetio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py index 7039ceb25a6d4..c990d3691ce96 100644 --- a/airbyte-integrations/connectors/source-freshcaller/main.py +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -4,5 +4,6 @@ from source_freshcaller.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py index b6757d75d1aaf..d4c01966794a9 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_freshcaller import SourceFreshcaller +from airbyte_cdk.entrypoint import launch + def run(): source = SourceFreshcaller() diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py index 61eb6c0174222..edfa285746a61 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshdesk/main.py b/airbyte-integrations/connectors/source-freshdesk/main.py index d32eaa6ca9e52..dc9d287ffe123 100644 --- a/airbyte-integrations/connectors/source-freshdesk/main.py +++ b/airbyte-integrations/connectors/source-freshdesk/main.py @@ -4,5 +4,6 @@ from source_freshdesk.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py index 3c87fa809e6a1..62c441218c17d 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement @@ -109,7 +110,9 @@ def get_request_params( start_time = stream_slice.get(self._partition_field_start.eval(self.config)) if not self.updated_slice else self.updated_slice options[self.start_time_option.field_name.eval(config=self.config)] = start_time # type: ignore # field_name is always casted to an interpolated string if self.end_time_option and self.end_time_option.inject_into == option_type: - options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(self._partition_field_end.eval(self.config)) # type: ignore # field_name is always casted to an interpolated string + options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get( + self._partition_field_end.eval(self.config) + ) # type: ignore # field_name is always casted to an interpolated string return options diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py index 982a82aa8163b..533d78205ce12 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_300_page.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models import SyncMode from conftest import find_stream +from airbyte_cdk.models import SyncMode + @pytest.fixture(name="responses") def responses_fixtures(): diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py index 619ad510a5065..cf598caf26290 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_incremental_sync.py @@ -2,9 +2,10 @@ import pytest -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from source_freshdesk.components import FreshdeskTicketsIncrementalSync +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType + class TestFreshdeskTicketsIncrementalSync: @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py index 4cdcf74fe6c1e..e5ac47ab81b5f 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_pagination_strategy.py @@ -8,7 +8,6 @@ class TestFreshdeskTicketsPaginationStrategy: - # returns None when there are fewer records than the page size @pytest.mark.parametrize( "response, current_page, last_records, expected", diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py index 1e5a1444574c3..2ca8dc18a9962 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py @@ -6,9 +6,10 @@ from typing import Any, MutableMapping import pytest +from conftest import find_stream + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream -from conftest import find_stream def _read_full_refresh(stream_instance: Stream): diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fullstory/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gcs/build_customization.py b/airbyte-integrations/connectors/source-gcs/build_customization.py index 83b1b136bf90d..33593a539450f 100644 --- a/airbyte-integrations/connectors/source-gcs/build_customization.py +++ b/airbyte-integrations/connectors/source-gcs/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py index 1fe90e1f13746..639577406d69f 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py @@ -14,6 +14,7 @@ from .utils import get_docker_ip + LOCAL_GCP_PORT = 4443 from urllib.parse import urlparse, urlunparse @@ -77,4 +78,3 @@ def connector_setup_fixture(docker_client) -> None: container.kill() container.remove() - diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py index e14dd36501e12..7046c434ca287 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/integration_test.py @@ -2,10 +2,11 @@ import os import pytest +from source_gcs import Config, SourceGCS, SourceGCSStreamReader + from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import read -from source_gcs import Config, SourceGCS, SourceGCSStreamReader from .utils import load_config diff --git a/airbyte-integrations/connectors/source-gcs/main.py b/airbyte-integrations/connectors/source-gcs/main.py index 695a41f39be33..3b58be6e85f08 100644 --- a/airbyte-integrations/connectors/source-gcs/main.py +++ b/airbyte-integrations/connectors/source-gcs/main.py @@ -5,5 +5,6 @@ from source_gcs.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py index 40c5ec5a5cf88..f2d3d37326f57 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py @@ -5,9 +5,10 @@ from typing import Literal, Union +from pydantic.v1 import AnyUrl, BaseModel, Field + from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig -from pydantic.v1 import AnyUrl, BaseModel, Field class OAuthCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py index 7b497a6a9f350..4dfaddb5c4d51 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py @@ -5,10 +5,11 @@ import json -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from google.cloud import storage from google.oauth2 import credentials, service_account +from airbyte_cdk.sources.file_based.remote_file import RemoteFile + def get_gcs_client(config): if config.credentials.auth_type == "Service": diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py index 65038c7fa7efd..c0a00bf919393 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/run.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/run.py @@ -7,9 +7,10 @@ import time import traceback +from orjson import orjson + from airbyte_cdk import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader from source_gcs.config_migrations import MigrateServiceAccount diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py index abc91843a4491..b81e60e95a32f 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py @@ -3,9 +3,10 @@ # from typing import Literal, Union -from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic.v1 import BaseModel, Field +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig + class OAuthCredentials(BaseModel): class Config(OneOfOptionConfig): diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py index fdd339c434249..53597e9316407 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py @@ -11,14 +11,16 @@ import pytz import smart_open -from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from google.cloud import storage from google.oauth2 import credentials, service_account + +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from source_gcs.config import Config from source_gcs.helpers import GCSRemoteFile from source_gcs.zip_helper import ZipHelper + # google can raise warnings for end user credentials, wrapping it to Logger logging.captureWarnings(True) @@ -96,7 +98,6 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo last_modified = blob.updated.astimezone(pytz.utc).replace(tzinfo=None) if not start_date or last_modified >= start_date: - if self.config.credentials.auth_type == "Client": uri = f"gs://{blob.bucket.name}/{blob.name}" else: diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py b/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py index 43b7c2deeecf5..609fc2e013dc8 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/zip_helper.py @@ -7,8 +7,10 @@ from typing import Iterable from google.cloud.storage.blob import Blob + from source_gcs.helpers import GCSRemoteFile + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py index 105a55e1ee085..5b72f16041645 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py @@ -6,10 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from source_gcs import Cursor, SourceGCSStreamReader from source_gcs.helpers import GCSRemoteFile +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig + @pytest.fixture def logger(): @@ -63,9 +64,10 @@ def zip_file(): uri=str(Path(__file__).parent / "resource/files/test.csv.zip"), last_modified=datetime.today(), mime_type=".zip", - displayed_uri="resource/files/test.csv.zip" + displayed_uri="resource/files/test.csv.zip", ) + @pytest.fixture def mocked_blob(): blob = Mock() diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py index ea301e8d9e12c..eab24e5e8d145 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py @@ -7,4 +7,3 @@ def test_documentation_url(): assert "https" in Config.documentation_url() - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py index 142a7f8939c8e..8812a03243eea 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py @@ -6,33 +6,42 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk import AirbyteEntrypoint from source_gcs import SourceGCS from source_gcs.config_migrations import MigrateServiceAccount +from airbyte_cdk import AirbyteEntrypoint + def load_config(path: str) -> Mapping[str, Any]: with open(path, "r") as f: return json.load(f) + def revert_config(path: str) -> None: migrated_config = load_config(path) del migrated_config["credentials"] with open(path, "w") as f: f.write(json.dumps(migrated_config)) + @pytest.mark.parametrize( "config_file_path, run_revert", [ # Migration is required (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_config.json"), True), # New config format - (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False) - ] + (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False), + ], ) def test_migrate_config(config_file_path, run_revert): args = ["check", "--config", config_file_path] - source = SourceGCS(MagicMock(), MagicMock, None, AirbyteEntrypoint.extract_config(args), None,) + source = SourceGCS( + MagicMock(), + MagicMock, + None, + AirbyteEntrypoint.extract_config(args), + None, + ) MigrateServiceAccount().migrate(args, source) migrated_config = load_config(config_file_path) @@ -43,4 +52,3 @@ def test_migrate_config(config_file_path, run_revert): if run_revert: revert_config(config_file_path) - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py index 59095eead97d8..309f5167a14a5 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py @@ -1,9 +1,10 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. from datetime import datetime +from source_gcs import Cursor + from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor -from source_gcs import Cursor def test_add_file_successfully(cursor, remote_file, logger): @@ -125,9 +126,6 @@ def test_add_file_zip_files(mocked_reader, zip_file, logger): cursor = Cursor(stream_config=FileBasedStreamConfig(name="test", globs=["**/*.zip"], format={"filetype": "csv"})) cursor.add_file(zip_file) - saved_history_cursor = datetime.strptime( - cursor._file_to_datetime_history[zip_file.displayed_uri], - cursor.DATE_TIME_FORMAT - ) + saved_history_cursor = datetime.strptime(cursor._file_to_datetime_history[zip_file.displayed_uri], cursor.DATE_TIME_FORMAT) assert saved_history_cursor == zip_file.last_modified diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py index 87b46cc061aa8..a74903a226b3f 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_run.py @@ -3,10 +3,11 @@ from unittest.mock import patch import pytest -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from common import catalog_path, config_path from source_gcs.run import run +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + def test_run_with_non_existing_config(): with patch("sys.argv", ["", "check", "--config", "non_existing.json"]): diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py index 03062a560865f..76c55728c0afc 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_source.py @@ -4,11 +4,12 @@ from unittest.mock import Mock import pytest -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.sources.file_based.availability_strategy import DefaultFileBasedAvailabilityStrategy from common import catalog_path, config_path from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.sources.file_based.availability_strategy import DefaultFileBasedAvailabilityStrategy + def _source_gcs(catalog, config): return SourceGCS( diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py index 163aaab29bf17..9ef22105964b9 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py @@ -9,8 +9,15 @@ def test_transform_record(zip_file, mocked_reader, logger): stream = GCSStream( - config=Mock(), catalog_schema=Mock(), stream_reader=Mock(), availability_strategy=Mock(), discovery_policy=Mock(),parsers=Mock(), - validation_policy=Mock(), errors_collector=Mock(), cursor=Mock() + config=Mock(), + catalog_schema=Mock(), + stream_reader=Mock(), + availability_strategy=Mock(), + discovery_policy=Mock(), + parsers=Mock(), + validation_policy=Mock(), + errors_collector=Mock(), + cursor=Mock(), ) last_updated = zip_file.last_modified.isoformat() transformed_record = stream.transform_record({"field1": 1}, zip_file, last_updated) @@ -19,11 +26,7 @@ def test_transform_record(zip_file, mocked_reader, logger): assert transformed_record["_ab_source_file_url"] != zip_file.uri last_updated = datetime.today().isoformat() - csv_file = GCSRemoteFile( - uri="https://storage.googleapis.com/test/test", - last_modified=last_updated, - mime_type = "csv" - ) + csv_file = GCSRemoteFile(uri="https://storage.googleapis.com/test/test", last_modified=last_updated, mime_type="csv") transformed_record = stream.transform_record({"field1": 1}, csv_file, last_updated) assert transformed_record["_ab_source_file_url"] == csv_file.uri diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py index b5d5b501872e4..b9508c8f5dde4 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py @@ -4,19 +4,17 @@ from unittest.mock import Mock import pytest +from source_gcs import Config, SourceGCSStreamReader +from source_gcs.config import ServiceAccountCredentials + from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from source_gcs import Config, SourceGCSStreamReader -from source_gcs.config import ServiceAccountCredentials def test_get_matching_files_with_no_prefix(logger, mocked_reader): mocked_reader._config = Config( - credentials=ServiceAccountCredentials( - service_account='{"type": "service_account"}', - auth_type="Service" - ), + credentials=ServiceAccountCredentials(service_account='{"type": "service_account"}', auth_type="Service"), bucket="test_bucket", streams=[], ) diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py index d4ea6e2fb08f9..f2595830b6db9 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py @@ -6,7 +6,6 @@ def test_get_gcs_remote_files(mocked_blob, zip_file, caplog): - files = list(ZipHelper(mocked_blob, zip_file,tempfile.TemporaryDirectory()).get_gcs_remote_files()) + files = list(ZipHelper(mocked_blob, zip_file, tempfile.TemporaryDirectory()).get_gcs_remote_files()) assert len(files) == 1 assert "Picking up file test.csv from zip archive" in caplog.text - diff --git a/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-genesys/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-genesys/main.py b/airbyte-integrations/connectors/source-genesys/main.py index d34643d2aa219..fbd608c13d5ae 100644 --- a/airbyte-integrations/connectors/source-genesys/main.py +++ b/airbyte-integrations/connectors/source-genesys/main.py @@ -4,5 +4,6 @@ from source_genesys.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py b/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py index 4a5adde124bcd..67fae1b42130f 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/authenicator.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index 30d0d1106c43d..27f2a07b52738 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -244,7 +245,6 @@ def path(self, **kwargs) -> str: class SourceGenesys(AbstractSource): def build_refresh_request_body(self) -> Mapping[str, Any]: - return { "grant_type": "client_credentials", "client_id": self.get_client_id(), @@ -259,7 +259,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - GENESYS_REGION_DOMAIN_MAP: Dict[str, str] = { "Americas (US East)": "mypurecloud.com", "Americas (US East 2)": "use2.us-gov-pure.cloud", diff --git a/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-getlago/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-github/fixtures/github.py b/airbyte-integrations/connectors/source-github/fixtures/github.py index 920b0677f9520..ce0eea2a046ce 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/github.py +++ b/airbyte-integrations/connectors/source-github/fixtures/github.py @@ -9,6 +9,7 @@ import requests + logging.basicConfig(level=logging.INFO) diff --git a/airbyte-integrations/connectors/source-github/fixtures/main.py b/airbyte-integrations/connectors/source-github/fixtures/main.py index 00468b290d426..47776cb046260 100644 --- a/airbyte-integrations/connectors/source-github/fixtures/main.py +++ b/airbyte-integrations/connectors/source-github/fixtures/main.py @@ -8,6 +8,7 @@ from github import GitHubFiller + if __name__ == "__main__": api_token = sys.argv[1] repository = sys.argv[2] diff --git a/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-github/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py index 4d37ce6cccf59..bbe2a227e2f8c 100644 --- a/airbyte-integrations/connectors/source-github/main.py +++ b/airbyte-integrations/connectors/source-github/main.py @@ -4,5 +4,6 @@ from source_github.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py b/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py index ac887892499c3..f407b315b21fd 100644 --- a/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py +++ b/airbyte-integrations/connectors/source-github/source_github/backoff_strategies.py @@ -6,6 +6,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py index 79ec73a9cd2f5..f644b4c452661 100644 --- a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py +++ b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceGithub + logger = logging.getLogger("airbyte_logger") @@ -30,13 +31,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: @@ -95,12 +94,10 @@ def migrate(cls, args: List[str], source: SourceGithub) -> None: class MigrateRepository(MigrateStringToArray): - migrate_from_key: str = "repository" migrate_to_key: str = "repositories" class MigrateBranch(MigrateStringToArray): - migrate_from_key: str = "branch" migrate_to_key: str = "branches" diff --git a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py index 26abc891e897c..ba36daec8ced9 100644 --- a/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py +++ b/airbyte-integrations/connectors/source-github/source_github/errors_handlers.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING @@ -12,6 +13,7 @@ from . import constants + GITHUB_DEFAULT_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 401: ErrorResolution( response_action=ResponseAction.RETRY, diff --git a/airbyte-integrations/connectors/source-github/source_github/github_schema.py b/airbyte-integrations/connectors/source-github/source_github/github_schema.py index ace0d9c69e2d2..bfa26d90dd379 100644 --- a/airbyte-integrations/connectors/source-github/source_github/github_schema.py +++ b/airbyte-integrations/connectors/source-github/source_github/github_schema.py @@ -6,6 +6,7 @@ import sgqlc.types.datetime import sgqlc.types.relay + github_schema = sgqlc.types.Schema() diff --git a/airbyte-integrations/connectors/source-github/source_github/graphql.py b/airbyte-integrations/connectors/source-github/source_github/graphql.py index 603e58f0182c1..bdeb4655bcd86 100644 --- a/airbyte-integrations/connectors/source-github/source_github/graphql.py +++ b/airbyte-integrations/connectors/source-github/source_github/graphql.py @@ -11,6 +11,7 @@ from . import github_schema + _schema = github_schema _schema_root = _schema.github_schema @@ -165,7 +166,6 @@ def get_query_issue_reactions(owner, name, first, after, number=None): class QueryReactions: - # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request. # If we try to fetch too many (up to 100) we will spend too many scores of query cost. # https://docs.github.com/en/graphql/overview/resource-limitations#calculating-a-rate-limit-score-before-running-the-call diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 5698b8795833d..9e6246e67ff35 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -60,7 +60,6 @@ class SourceGithub(AbstractSource): - continue_sync_on_stream_failure = True @staticmethod diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 7e9fcc80cc3e9..cc8d8d8ee2b2a 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk import BackoffStrategy, StreamSlice from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode from airbyte_cdk.models import Type as MessageType @@ -41,7 +42,6 @@ class GithubStreamABC(HttpStream, ABC): - primary_key = "id" # Detect streams with high API load @@ -80,7 +80,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} if next_page_token: @@ -756,7 +755,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class GitHubGraphQLStream(GithubStream, ABC): - http_method = "POST" def path( @@ -976,7 +974,6 @@ def request_body_json( class ReactionStream(GithubStream, CheckpointMixin, ABC): - parent_key = "id" copy_parent_key = "comment_id" cursor_field = "created_at" @@ -1394,9 +1391,9 @@ def _get_updated_state(self, current_stream_state: MutableMapping[str, Any], lat stream_state_value = current_stream_state.get(repository, {}).get(project_id, {}).get(column_id, {}).get(self.cursor_field) if stream_state_value: updated_state = max(updated_state, stream_state_value) - current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[ - self.cursor_field - ] = updated_state + current_stream_state.setdefault(repository, {}).setdefault(project_id, {}).setdefault(column_id, {})[self.cursor_field] = ( + updated_state + ) return current_stream_state def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: @@ -1621,7 +1618,6 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, return record def get_error_handler(self) -> Optional[ErrorHandler]: - return ContributorActivityErrorHandler(logger=self.logger, max_retries=5, error_mapping=GITHUB_DEFAULT_ERROR_MAPPING) def get_backoff_strategy(self) -> Optional[Union[BackoffStrategy, List[BackoffStrategy]]]: diff --git a/airbyte-integrations/connectors/source-github/source_github/utils.py b/airbyte-integrations/connectors/source-github/source_github/utils.py index 3479e7c12b437..d9b7ea18d3bcb 100644 --- a/airbyte-integrations/connectors/source-github/source_github/utils.py +++ b/airbyte-integrations/connectors/source-github/source_github/utils.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator @@ -99,7 +100,6 @@ def update_token(self) -> None: @property def token(self) -> str: - token = self.current_active_token return f"{self._auth_method} {token}" @@ -123,13 +123,15 @@ def _check_token_limits(self, token: str): ) token_info = self._tokens[token] remaining_info_core = rate_limit_info.get("core") - token_info.count_rest, token_info.reset_at_rest = remaining_info_core.get("remaining"), pendulum.from_timestamp( - remaining_info_core.get("reset") + token_info.count_rest, token_info.reset_at_rest = ( + remaining_info_core.get("remaining"), + pendulum.from_timestamp(remaining_info_core.get("reset")), ) remaining_info_graphql = rate_limit_info.get("graphql") - token_info.count_graphql, token_info.reset_at_graphql = remaining_info_graphql.get("remaining"), pendulum.from_timestamp( - remaining_info_graphql.get("reset") + token_info.count_graphql, token_info.reset_at_graphql = ( + remaining_info_graphql.get("remaining"), + pendulum.from_timestamp(remaining_info_graphql.get("reset")), ) def check_all_tokens(self): diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index f4e454dfab989..b2f4f27fe7a55 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -5,6 +5,7 @@ import pytest import responses + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py index bde678baef18a..89c9b35fcc88b 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py @@ -3,16 +3,18 @@ import json from unittest import TestCase +from source_github import SourceGithub + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from source_github import SourceGithub from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_repositories(["airbytehq/mock-test-0", "airbytehq/mock-test-1", "airbytehq/mock-test-2"]).build() @@ -162,12 +164,19 @@ def test_read_full_refresh_emits_per_partition_state(self): per_partition_state_1 = {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}} per_partition_state_2 = {"partition": {"repository": "airbytehq/mock-test-2"}, "cursor": {"__ab_full_refresh_sync_complete": True}} - incoming_state = StateBuilder().with_stream_state("assignees", { - "states": [ - {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - ] - }).build() + incoming_state = ( + StateBuilder() + .with_stream_state( + "assignees", + { + "states": [ + {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + ] + }, + ) + .build() + ) source = SourceGithub() actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), state=incoming_state) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py index 4140e6a1b63dd..dfea868aed687 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py @@ -3,6 +3,8 @@ import json from unittest import TestCase, mock +from source_github import SourceGithub + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -10,10 +12,10 @@ from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import AirbyteStreamStatus, Level, TraceType -from source_github import SourceGithub from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_repositories(["airbytehq/integration-test"]).build() diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py index fdebc3af470bb..06ad40cf27561 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py @@ -7,11 +7,13 @@ import os from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_github.config_migrations import MigrateBranch, MigrateRepository from source_github.source import SourceGithub +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py index ebe8eb56c4a4a..829c6137eb953 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py @@ -8,13 +8,14 @@ import pendulum import pytest import responses -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType from freezegun import freeze_time from source_github import SourceGithub from source_github.streams import Organizations from source_github.utils import MultipleTokenAuthenticatorWithRateLimiter, read_full_refresh +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + @responses.activate def test_multiple_tokens(rate_limit_mock_response): diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 388e726e7dde0..8e0d2455d0fcd 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -8,11 +8,12 @@ import pytest import responses -from airbyte_cdk.models import AirbyteConnectionStatus, Status -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_github import constants from source_github.source import SourceGithub +from airbyte_cdk.models import AirbyteConnectionStatus, Status +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + from .utils import command_check diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index d8944137c766f..1ee7c865bbfb2 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -10,10 +10,6 @@ import pytest import requests import responses -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction -from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException -from airbyte_protocol.models import FailureType from requests import HTTPError from responses import matchers from source_github import SourceGithub, constants @@ -55,8 +51,14 @@ ) from source_github.utils import read_full_refresh +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, HttpStatusErrorHandler, ResponseAction +from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException +from airbyte_protocol.models import FailureType + from .utils import ProjectsResponsesAPI, read_incremental + DEFAULT_BACKOFF_DELAYS = [1, 2, 4, 8, 16] @@ -102,10 +104,34 @@ def test_backoff_time(time_mock, http_status, response_headers, expected_backoff @pytest.mark.parametrize( ("http_status", "response_headers", "text", "response_action", "error_message"), [ - (HTTPStatus.OK, {"X-RateLimit-Resource": "graphql"}, '{"errors": [{"type": "RATE_LIMITED"}]}', ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.OK}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"X-RateLimit-Remaining": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "60"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), + ( + HTTPStatus.OK, + {"X-RateLimit-Resource": "graphql"}, + '{"errors": [{"type": "RATE_LIMITED"}]}', + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.OK}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"X-RateLimit-Remaining": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "60"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), (HTTPStatus.INTERNAL_SERVER_ERROR, {}, "", ResponseAction.RETRY, "Internal server error."), (HTTPStatus.BAD_GATEWAY, {}, "", ResponseAction.RETRY, "Bad gateway."), (HTTPStatus.SERVICE_UNAVAILABLE, {}, "", ResponseAction.RETRY, "Service unavailable."), @@ -330,7 +356,6 @@ def test_stream_repositories_read(): @responses.activate @patch("time.sleep") def test_stream_projects_disabled(time_mock): - repository_args_with_start_date = {"start_date": "start_date", "page_size_for_large_streams": 30, "repositories": ["test_repo"]} stream = Projects(**repository_args_with_start_date) @@ -348,7 +373,6 @@ def test_stream_projects_disabled(time_mock): @responses.activate def test_stream_pull_requests_incremental_read(): - page_size = 2 repository_args_with_start_date = { "repositories": ["organization/repository"], @@ -412,7 +436,6 @@ def test_stream_pull_requests_incremental_read(): @responses.activate def test_stream_commits_incremental_read(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -451,18 +474,18 @@ def test_stream_commits_incremental_read(): "name": "branch", "commit": { "sha": "74445338726f0f8e1c27c10dce90ca00c5ae2858", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858", }, - "protected": False + "protected": False, }, { "name": "main", "commit": { "sha": "c27c10dce90ca00c5ae285874445338726f0f8e1", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1", }, - "protected": False - } + "protected": False, + }, ], status=200, ) @@ -503,7 +526,6 @@ def test_stream_commits_incremental_read(): @responses.activate def test_stream_pull_request_commits(): - repository_args = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -545,7 +567,6 @@ def test_stream_pull_request_commits(): @responses.activate def test_stream_project_columns(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -638,7 +659,6 @@ def test_stream_project_columns(): @responses.activate def test_stream_project_cards(): - repository_args_with_start_date = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -734,7 +754,6 @@ def test_stream_project_cards(): @responses.activate def test_stream_comments(): - repository_args_with_start_date = { "repositories": ["organization/repository", "airbytehq/airbyte"], "page_size_for_large_streams": 2, @@ -866,7 +885,6 @@ def test_stream_comments(): @responses.activate def test_streams_read_full_refresh(): - repository_args = { "repositories": ["organization/repository"], "page_size_for_large_streams": 100, @@ -927,7 +945,6 @@ def get_records(cursor_field): @responses.activate def test_stream_reviews_incremental_read(): - repository_args_with_start_date = { "start_date": "2000-01-01T00:00:00Z", "page_size_for_large_streams": 30, @@ -1002,7 +1019,6 @@ def test_stream_team_members_full_refresh(time_mock, caplog, rate_limit_mock_res @responses.activate def test_stream_commit_comment_reactions_incremental_read(): - repository_args = {"repositories": ["airbytehq/integration-test"], "page_size_for_large_streams": 100} stream = CommitCommentReactions(**repository_args) stream._parent_stream._http_client._session.cache.clear() @@ -1083,7 +1099,6 @@ def test_stream_commit_comment_reactions_incremental_read(): @responses.activate def test_stream_workflow_runs_read_incremental(monkeypatch): - repository_args_with_start_date = { "repositories": ["org/repos"], "page_size_for_large_streams": 30, @@ -1202,7 +1217,6 @@ def test_stream_workflow_runs_read_incremental(monkeypatch): @responses.activate def test_stream_workflow_jobs_read(): - repository_args = { "repositories": ["org/repo"], "page_size_for_large_streams": 100, @@ -1303,7 +1317,6 @@ def test_stream_workflow_jobs_read(): @responses.activate def test_stream_pull_request_comment_reactions_read(): - repository_args_with_start_date = { "start_date": "2022-01-01T00:00:00Z", "page_size_for_large_streams": 2, @@ -1361,11 +1374,7 @@ def test_stream_projects_v2_graphql_retry(time_mock, rate_limit_mock_response): } stream = ProjectsV2(**repository_args_with_start_date) resp = responses.add( - responses.POST, - "https://api.github.com/graphql", - json={"errors": "not found"}, - status=200, - headers={'Retry-After': '5'} + responses.POST, "https://api.github.com/graphql", json={"errors": "not found"}, status=200, headers={"Retry-After": "5"} ) backoff_strategy = GithubStreamABCBackoffStrategy(stream) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/utils.py b/airbyte-integrations/connectors/source-github/unit_tests/utils.py index 81b32a929bcf4..77472173550b8 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/utils.py @@ -6,6 +6,7 @@ from unittest import mock import responses + from airbyte_cdk.models import SyncMode from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification from airbyte_cdk.sources import Source diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gitlab/main.py b/airbyte-integrations/connectors/source-gitlab/main.py index 1c322c2f2c48a..8dcc7a60745fe 100644 --- a/airbyte-integrations/connectors/source-gitlab/main.py +++ b/airbyte-integrations/connectors/source-gitlab/main.py @@ -4,5 +4,6 @@ from source_gitlab.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py index ec47f547f591c..1640cc68ad998 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py @@ -12,6 +12,7 @@ from .source import SourceGitlab + logger = logging.getLogger("airbyte_logger") @@ -30,13 +31,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py index c6f180e38de9b..1b4a6d60636d3 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -6,9 +6,11 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.sources.streams import Stream from source_gitlab.source import SourceGitlab +from airbyte_cdk.sources.streams import Stream + + BASE_CONFIG = { "start_date": "2021-01-01T00:00:00Z", "api_url": "gitlab.com", diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py index b61fb232906d2..4293f2ed46951 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py @@ -5,6 +5,7 @@ from source_gitlab.config_migrations import MigrateGroups from source_gitlab.source import SourceGitlab + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py index 4e7e0b40483fc..7222619f32cc6 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py @@ -3,9 +3,10 @@ # +from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name class TestGroupStreamsPartitionRouter: @@ -56,9 +57,7 @@ def test_projects_stream_slices_with_group_project_ids(self, requests_mock): url=f"https://gitlab.com/api/v4/groups/{group_id}?per_page=50", json=[{"id": group_id, "projects": [{"id": project_id, "path_with_namespace": project_id}]}], ) - expected_stream_slices.append( - StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={}) - ) + expected_stream_slices.append(StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})) requests_mock.get(url=GROUPS_LIST_URL, json=groups_list_response) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index b650c9e52b6d8..06f55edf7d51f 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -6,9 +6,10 @@ import logging import pytest -from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from source_gitlab import SourceGitlab +from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream + def test_streams(config): source = SourceGitlab() @@ -19,9 +20,7 @@ def test_streams(config): def test_connection_success(config, requests_mock): requests_mock.get(url="/api/v4/groups", json=[{"id": "g1"}]) - requests_mock.get( - url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}] - ) + requests_mock.get(url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]) requests_mock.get(url="/api/v4/projects/p1", json={"id": "p1"}) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) @@ -30,9 +29,7 @@ def test_connection_success(config, requests_mock): def test_connection_invalid_projects_and_projects(config_with_project_groups, requests_mock): requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1?per_page=50", status_code=404) - requests_mock.register_uri( - "GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404 - ) + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404) requests_mock.register_uri("GET", "https://gitlab.com/api/v4/projects/p1?per_page=50&statistics=1", status_code=404) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config_with_project_groups) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index 9eef564ef235b..820ca47f17b5f 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -4,9 +4,11 @@ import pytest -from airbyte_cdk.models import SyncMode from conftest import BASE_CONFIG, GROUPS_LIST_URL, get_stream_by_name +from airbyte_cdk.models import SyncMode + + CONFIG = BASE_CONFIG | {"projects_list": ["p_1"]} @@ -123,7 +125,7 @@ def test_should_retry(requests_mock, stream_name, extra_mocks): "user_full_name": "John", "environment_name": "dev", "project_id": "p_1", - } + }, ), ( "merge_request_commits", @@ -157,11 +159,12 @@ def test_stream_slices_child_stream(requests_mock): url="https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1", json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}], ) - stream_state = {"13082000": {""'created_at': "2021-03-10T23:58:1213"}} + stream_state = {"13082000": {"" "created_at": "2021-03-10T23:58:1213"}} slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state)) assert slices + def test_request_params(): commits = get_stream_by_name("commits", CONFIG) assert commits.retriever.requester.get_request_params() == {"with_stats": "true"} diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py index 5019d87b7b44c..a347efae2f82c 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py @@ -11,7 +11,7 @@ ("http://example", (True, "http", "example")), ("test://example.com", (False, "", "")), ("https://example.com/test/test2", (False, "", "")), - ) + ), ) def test_parse_url(url, expected): assert parse_url(url) == expected diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gnews/components.py b/airbyte-integrations/connectors/source-gnews/components.py index df2f0a450bdad..d93c3d9631746 100644 --- a/airbyte-integrations/connectors/source-gnews/components.py +++ b/airbyte-integrations/connectors/source-gnews/components.py @@ -7,6 +7,7 @@ from typing import Any, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.requesters.error_handlers import BackoffStrategy from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gnews/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gocardless/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-goldcast/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gong/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py index f2b023043a0b8..de602c5991481 100644 --- a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py +++ b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py @@ -4,9 +4,10 @@ from json import dumps, load from typing import Dict +from components import IncrementalSingleBodyFilterCursor + from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption -from components import IncrementalSingleBodyFilterCursor def config() -> Dict[str, str]: @@ -17,35 +18,30 @@ def config() -> Dict[str, str]: class TestGetRequestFilterOptions(unittest.TestCase): - def setUp(self): self.instance = IncrementalSingleBodyFilterCursor( - start_datetime= MinMaxDatetime(datetime= "2024-03-25", datetime_format= "%Y-%m-%dT%H:%M:%SZ", parameters = None), - cursor_field = "reviewTime", - datetime_format= "%Y-%m-%d", - config= config, - parameters = None + start_datetime=MinMaxDatetime(datetime="2024-03-25", datetime_format="%Y-%m-%dT%H:%M:%SZ", parameters=None), + cursor_field="reviewTime", + datetime_format="%Y-%m-%d", + config=config, + parameters=None, ) - + def test_get_request_filter_options_no_stream_slice(self): expected = {} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, None) assert result == expected - + def test_get_request_filter_options_with_start_time_option(self): - expected = {'filter': {'reviewFromDate': '2024-03-25'}} - - self.instance.start_time_option = RequestOption( - inject_into = "body_json", - field_name = "filter, reviewFromDate", - parameters = None - ) - self.instance.stream_slice = {'start_time': '2024-03-25', 'end_time': '2024-03-29'} + expected = {"filter": {"reviewFromDate": "2024-03-25"}} + + self.instance.start_time_option = RequestOption(inject_into="body_json", field_name="filter, reviewFromDate", parameters=None) + self.instance.stream_slice = {"start_time": "2024-03-25", "end_time": "2024-03-29"} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, self.instance.stream_slice) assert result == expected -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py index c6960355b90c6..aa6c453f90180 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_test.py @@ -6,10 +6,11 @@ from pathlib import Path import pytest -from airbyte_cdk.models import SyncMode from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow from source_google_ads.source import SourceGoogleAds +from airbyte_cdk.models import SyncMode + @pytest.fixture(scope="module") def config(): diff --git a/airbyte-integrations/connectors/source-google-ads/main.py b/airbyte-integrations/connectors/source-google-ads/main.py index 2824c49559439..7e8e4ddf52260 100644 --- a/airbyte-integrations/connectors/source-google-ads/main.py +++ b/airbyte-integrations/connectors/source-google-ads/main.py @@ -4,5 +4,6 @@ from source_google_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py index 81bae9ceb5cb7..61990be3b7b95 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py @@ -14,6 +14,7 @@ from .utils import GAQL + FULL_REFRESH_CUSTOM_TABLE = [ "asset", "asset_group_listing_group_filter", diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 4a3ac096cfdd6..4b2a25a08a32e 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -9,6 +9,7 @@ from .streams import GoogleAdsStream, IncrementalGoogleAdsStream from .utils import GAQL + DATE_TYPES = ("segments.date", "segments.month", "segments.quarter", "segments.week") diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 3a2128f6182ff..9822100667667 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -7,8 +7,6 @@ from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping import backoff -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.client import GoogleAdsClient from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse from google.api_core.exceptions import InternalServerError, ServerError, TooManyRequests @@ -17,8 +15,12 @@ from google.protobuf.message import Message from proto.marshal.collections import Repeated, RepeatedComposite +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + from .utils import logger + API_VERSION = "v17" diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 562dacf6f8992..4fa488df17e33 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -6,11 +6,12 @@ import logging from typing import Any, Iterable, List, Mapping, MutableMapping, Tuple +from pendulum import duration, parse, today + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from pendulum import duration, parse, today from .custom_query_stream import CustomQuery, IncrementalCustomQuery from .google_ads import GoogleAds diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 0dc3374548fae..99da0e71f661b 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -8,15 +8,16 @@ import backoff import pendulum -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.sources.streams import CheckpointMixin, Stream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.services.services.google_ads_service.pagers import SearchPager from google.ads.googleads.v17.services.types.google_ads_service import SearchGoogleAdsResponse from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests, Unauthenticated +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.sources.streams import CheckpointMixin, Stream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException + from .google_ads import GoogleAds, logger from .models import CustomerModel from .utils import ExpiredPageTokenError, chunk_date_range, detached, generator_backoff, get_resource_name, parse_dates, traced_exception diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index 824d9c9cb470e..5b31eac22e37e 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -13,8 +13,6 @@ from typing import Any, Callable, Generator, Iterable, MutableMapping, Optional, Tuple, Type, Union import pendulum -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.errors.types.authentication_error import AuthenticationErrorEnum from google.ads.googleads.v17.errors.types.authorization_error import AuthorizationErrorEnum @@ -23,6 +21,10 @@ from google.ads.googleads.v17.errors.types.request_error import RequestErrorEnum from google.api_core.exceptions import Unauthenticated +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException + + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py index f409e0f1ac52a..21623048feda0 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -55,6 +55,7 @@ def mock_oauth_call(requests_mock): def customers(config): return [CustomerModel(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + @pytest.fixture def additional_customers(config, customers): return customers + [CustomerModel(id="789", time_zone="local", is_manager_account=False)] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py index a65712bf0e9d3..c6a7e287ed07f 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_ads.config_migrations import MigrateCustomQuery from source_google_ads.source import SourceGoogleAds +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py index 49679c3be829b..bb99b9cc427f3 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -8,12 +8,13 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads.google_ads import GoogleAds from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds from source_google_ads.streams import AdGroupLabel, Label, ServiceAccounts +from airbyte_cdk.utils import AirbyteTracedException + from .common import MockGoogleAdsClient, mock_google_ads_request_failure @@ -35,7 +36,10 @@ def mock_get_customers(mocker): "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", ), (["QUERY_ERROR"], "Incorrect custom query. Error in query: unexpected end of query."), - (["UNRECOGNIZED_FIELD"], "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid."), + ( + ["UNRECOGNIZED_FIELD"], + "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid.", + ), ( ["RESOURCE_EXHAUSTED"], ( diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index e3bec07f29762..5c8f93b4e7cfa 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -8,14 +8,16 @@ import pendulum import pytest -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.v17.services.types.google_ads_service import GoogleAdsRow from google.auth import exceptions from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import chunk_date_range +from airbyte_cdk.utils import AirbyteTracedException + from .common import MockGoogleAdsClient, MockGoogleAdsService + SAMPLE_SCHEMA = { "properties": { "segment.date": { @@ -148,11 +150,12 @@ def test_get_field_value(): response = GoogleAds.get_field_value(MockedDateSegment(date), field, {}) assert response == date + def test_get_field_value_object(): expected_response = [ - { "text": "An exciting headline", "policySummaryInfo": { "reviewStatus": "REVIEWED","approvalStatus":"APPROVED" } }, - { "text": "second" }, - ] + {"text": "An exciting headline", "policySummaryInfo": {"reviewStatus": "REVIEWED", "approvalStatus": "APPROVED"}}, + {"text": "second"}, + ] field = "ad_group_ad.ad.responsive_search_ad.headlines" ads_row = GoogleAdsRow( ad_group_ad={ @@ -161,14 +164,9 @@ def test_get_field_value_object(): "headlines": [ { "text": "An exciting headline", - "policy_summary_info": { - "review_status": "REVIEWED", - "approval_status": "APPROVED" - } + "policy_summary_info": {"review_status": "REVIEWED", "approval_status": "APPROVED"}, }, - { - "text": "second" - } + {"text": "second"}, ] } } @@ -176,13 +174,14 @@ def test_get_field_value_object(): ) response = GoogleAds.get_field_value(ads_row, field, {}) - assert [json.loads(i) for i in response] == expected_response + assert [json.loads(i) for i in response] == expected_response + def test_get_field_value_strings(): expected_response = [ - "http://url_one.com", - "https://url_two.com", - ] + "http://url_one.com", + "https://url_two.com", + ] ads_row = GoogleAdsRow( ad_group_ad={ "ad": { @@ -197,6 +196,7 @@ def test_get_field_value_strings(): response = GoogleAds.get_field_value(ads_row, field, {}) assert response == expected_response + def test_parse_single_result(): date = "2001-01-01" response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py index 34b89d1e2e881..50173bce7ff32 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py @@ -8,11 +8,12 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import CampaignCriterion, ChangeStatus +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException + def mock_response_parent(): yield [ @@ -82,7 +83,7 @@ def test_change_status_stream(config, customers): def test_change_status_stream_slices(config, additional_customers): - """ Change status stream slices should return correct empty slices for the new customers """ + """Change status stream slices should return correct empty slices for the new customers""" google_api = MockGoogleAds(credentials=config["credentials"]) stream = ChangeStatus(api=google_api, customers=additional_customers) @@ -94,12 +95,19 @@ def test_change_status_stream_slices(config, additional_customers): result_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(result_slices) == 2 - assert result_slices == [{'start_date': '2023-11-01 12:36:04.772447', 'end_date': '2023-11-02 00:00:00.000000', 'customer_id': '123', - 'login_customer_id': None}, {'customer_id': '789', 'login_customer_id': None}] + assert result_slices == [ + { + "start_date": "2023-11-01 12:36:04.772447", + "end_date": "2023-11-02 00:00:00.000000", + "customer_id": "123", + "login_customer_id": None, + }, + {"customer_id": "789", "login_customer_id": None}, + ] def test_incremental_events_stream_slices(config, additional_customers): - """ Test if the empty slice will be produced for the new customers """ + """Test if the empty slice will be produced for the new customers""" stream_state = {"change_status": {"123": {"change_status.last_change_date_time": "2023-06-12 13:20:01.003295"}}} google_api = MockGoogleAds(credentials=config["credentials"]) @@ -121,12 +129,21 @@ def test_incremental_events_stream_slices(config, additional_customers): stream_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(stream_slices) == 2 - assert stream_slices == [{'customer_id': '123', 'updated_ids': {'2', '1'}, 'deleted_ids': {'3', '4'}, - 'record_changed_time_map': {'1': '2023-06-13 12:36:01.772447', '2': '2023-06-13 12:36:02.772447', - '3': '2023-06-13 12:36:03.772447', '4': '2023-06-13 12:36:04.772447'}, - 'login_customer_id': None}, - {'customer_id': '789', 'updated_ids': set(), 'deleted_ids': set(), 'record_changed_time_map': {}, - 'login_customer_id': None}] + assert stream_slices == [ + { + "customer_id": "123", + "updated_ids": {"2", "1"}, + "deleted_ids": {"3", "4"}, + "record_changed_time_map": { + "1": "2023-06-13 12:36:01.772447", + "2": "2023-06-13 12:36:02.772447", + "3": "2023-06-13 12:36:03.772447", + "4": "2023-06-13 12:36:04.772447", + }, + "login_customer_id": None, + }, + {"customer_id": "789", "updated_ids": set(), "deleted_ids": set(), "record_changed_time_map": {}, "login_customer_id": None}, + ] def test_child_incremental_events_read(config, customers): diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index fd53878536cf8..10c384e532c9d 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -10,8 +10,6 @@ import pendulum import pytest -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, FailureType, SyncMode from pendulum import duration, today from source_google_ads.custom_query_stream import IncrementalCustomQuery from source_google_ads.google_ads import GoogleAds @@ -20,6 +18,9 @@ from source_google_ads.streams import AdGroupAdLegacy, chunk_date_range from source_google_ads.utils import GAQL +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, FailureType, SyncMode + from .common import MockGoogleAdsClient diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index d0665bea9ac92..fbf8c8f07f3c5 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -6,8 +6,6 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException from google.ads.googleads.v17.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure from google.ads.googleads.v17.errors.types.request_error import RequestErrorEnum @@ -16,6 +14,10 @@ from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import AdGroup, ClickView, Customer, CustomerLabel +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils import AirbyteTracedException + + # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired. exception = GoogleAdsException( error=RpcError(), @@ -251,8 +253,10 @@ def test_retry_500_raises_transient_error(mocker, config, customers): with pytest.raises(AirbyteTracedException) as exception: records = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) - assert exception.value.internal_message == ("Internal Error encountered Unable to fetch data from Google Ads API due to " - "temporal error on the Google Ads server. Please retry again later. ") + assert exception.value.internal_message == ( + "Internal Error encountered Unable to fetch data from Google Ads API due to " + "temporal error on the Google Ads server. Please retry again later. " + ) assert exception.value.failure_type == FailureType.transient_error assert mocked_search.call_count == 5 assert records == [] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py index fe162fb993bc3..5fbeab771444b 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -8,10 +8,11 @@ import backoff import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_google_ads import SourceGoogleAds from source_google_ads.utils import GAQL, generator_backoff +from airbyte_cdk.utils import AirbyteTracedException + def test_parse_GAQL_ok(): sql = GAQL.parse("SELECT field FROM table") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py index 93839ed0e51a5..6133031547675 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py @@ -4,5 +4,6 @@ from source_google_analytics_data_api.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py index 27591cb3a1a72..2267cd3a19e32 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py @@ -8,9 +8,10 @@ from typing import Any, Iterable, Mapping, Optional import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from requests.exceptions import JSONDecodeError +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + from .utils import API_LIMIT_PER_HOUR diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py index 4a4bedce6ac39..b290368c983fb 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/authenticator.py @@ -6,6 +6,7 @@ import jwt import requests + from source_google_analytics_data_api import utils diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py index c055b6445bc2e..e1d1d23ccf824 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py @@ -8,6 +8,7 @@ import dpath.util import orjson + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import AirbyteMessageSerializer @@ -15,6 +16,7 @@ from .source import SourceGoogleAnalyticsDataApi + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py index ea17f13864ab3..f3e4723157c76 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/google_analytics_data_api_metadata_error_mapping.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + PROPERTY_ID_DOCS_URL = "https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id" MESSAGE = "Incorrect Property ID: {property_id}. Access was denied to the property ID entered. Check your access to the Property ID or use Google Analytics {property_id_docs_url} to find your Property ID." diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 957805ecde5b2..1ed3e035984c7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -18,6 +18,8 @@ import jsonschema import pendulum import requests +from requests import HTTPError + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -28,7 +30,6 @@ from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException from airbyte_cdk.sources.streams.http.http_client import MessageRepresentationAirbyteTracedErrors from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError from source_google_analytics_data_api import utils from source_google_analytics_data_api.google_analytics_data_api_base_error_mapping import get_google_analytics_data_api_base_error_mapping from source_google_analytics_data_api.google_analytics_data_api_metadata_error_mapping import ( @@ -49,6 +50,7 @@ transform_json, ) + # set the quota handler globally since limitations are the same for all streams # the initial values should be saved once and tracked for each stream, inclusively. GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py index d2783f6634ba6..f08a5472c0d42 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py @@ -12,9 +12,11 @@ import jsonschema import pandas as pd + from airbyte_cdk.sources.streams.http import requests_native_auth as auth from source_google_analytics_data_api.authenticator import GoogleServiceKeyAuthenticator + DATE_FORMAT = "%Y-%m-%d" metrics_data_native_types_map: Dict = { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py index fcd8e1b879be3..95b944b3bae72 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pytest + # json credentials with fake private key json_credentials = """ { diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py index bd5af212c081f..af86742f72ebd 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py @@ -4,9 +4,11 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from source_google_analytics_data_api.api_quota import GoogleAnalyticsApiQuota +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + + TEST_QUOTA_INSTANCE: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() @@ -35,7 +37,7 @@ def test_check_initial_quota_is_empty(): "potentiallyThresholdedRequestsPerHour": {"consumed": 1, "remaining": 26}, } }, - False, # partial_quota + False, # partial_quota ResponseAction.RETRY, None, # backoff_time_exp False, # stop_iter_exp diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py index a0df1b3f59d2d..aa70ba13cb9d3 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py @@ -4,10 +4,11 @@ from unittest.mock import patch -from airbyte_cdk.entrypoint import AirbyteEntrypoint from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi from source_google_analytics_data_api.config_migrations import MigratePropertyID +from airbyte_cdk.entrypoint import AirbyteEntrypoint + @patch.object(SourceGoogleAnalyticsDataApi, "read_config") @patch.object(SourceGoogleAnalyticsDataApi, "write_config") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py index 340d6db7660ea..aac51ada0dcc5 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py @@ -8,11 +8,13 @@ from typing import Any, Mapping import dpath.util -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_analytics_data_api.config_migrations import MigrateCustomReportsCohortSpec from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py index 5bee2f8ab6b94..39b569b5f2342 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_analytics_data_api.config_migrations import MigrateCustomReports from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index e86201860e8bf..d2d801e9bb2d2 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -5,14 +5,15 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status -from airbyte_cdk.sources.streams.http.http import HttpStatusErrorHandler -from airbyte_cdk.utils import AirbyteTracedException from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi from source_google_analytics_data_api.api_quota import GoogleAnalyticsApiQuotaBase from source_google_analytics_data_api.source import GoogleAnalyticsDatApiErrorHandler, MetadataDescriptor from source_google_analytics_data_api.utils import NO_DIMENSIONS, NO_METRICS, NO_NAME, WRONG_CUSTOM_REPORT_CONFIG, WRONG_JSON_SYNTAX +from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status +from airbyte_cdk.sources.streams.http.http import HttpStatusErrorHandler +from airbyte_cdk.utils import AirbyteTracedException + @pytest.mark.parametrize( "config_values, is_successful, message", @@ -111,32 +112,42 @@ def test_check_failure_throws_exception(requests_mock, config_gen, error_code): def test_exhausted_quota_recovers_after_two_retries(requests_mock, config_gen): """ - If the account runs out of quota the api will return a message asking us to back off for one hour. - We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not - it will backoff again 30 minutes and quote should be reestablished by then. - Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota - recovery time and also the backoff time. + If the account runs out of quota the api will return a message asking us to back off for one hour. + We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not + it will backoff again 30 minutes and quote should be reestablished by then. + Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota + recovery time and also the backoff time. """ requests_mock.register_uri( "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} ) - error_response = {"error": {"message":"Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see"}} + error_response = { + "error": { + "message": "Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see" + } + } requests_mock.register_uri( "GET", "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata", # first try we get 429 t=~0 - [{"json": error_response, "status_code": 429}, - # first retry we get 429 t=~1800 - {"json": error_response, "status_code": 429}, - # second retry quota is recovered, t=~3600 - {"json": { - "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], - "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], - }, "status_code": 200} - ] + [ + {"json": error_response, "status_code": 429}, + # first retry we get 429 t=~1800 + {"json": error_response, "status_code": 429}, + # second retry quota is recovered, t=~3600 + { + "json": { + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + "status_code": 200, + }, + ], ) + def fix_time(time): - return int(time / 600 ) + return int(time / 600) + source = SourceGoogleAnalyticsDataApi() logger = MagicMock() max_time_fixed = fix_time(GoogleAnalyticsDatApiErrorHandler.QUOTA_RECOVERY_TIME) @@ -146,10 +157,18 @@ def fix_time(time): potentially_thresholded_requests_per_hour_mapping_fixed = { **potentially_thresholded_requests_per_hour_mapping, "backoff": fixed_threshold_backoff_time, - } + } with ( - patch.object(GoogleAnalyticsDatApiErrorHandler, 'QUOTA_RECOVERY_TIME', new=max_time_fixed), - patch.object(GoogleAnalyticsApiQuotaBase, 'quota_mapping', new={**GoogleAnalyticsApiQuotaBase.quota_mapping,"potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed})): + patch.object(GoogleAnalyticsDatApiErrorHandler, "QUOTA_RECOVERY_TIME", new=max_time_fixed), + patch.object( + GoogleAnalyticsApiQuotaBase, + "quota_mapping", + new={ + **GoogleAnalyticsApiQuotaBase.quota_mapping, + "potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed, + }, + ), + ): output = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert output == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) @@ -164,7 +183,7 @@ def test_check_failure(requests_mock, config_gen, error_code): ) source = SourceGoogleAnalyticsDataApi() logger = MagicMock() - with patch.object(HttpStatusErrorHandler, 'max_retries', new=0): + with patch.object(HttpStatusErrorHandler, "max_retries", new=0): airbyte_status = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert airbyte_status.status == Status.FAILED assert airbyte_status.message == repr("Failed to get metadata, over quota, try later") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 3ab018bbe6d35..97cb79d8b64de 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -10,11 +10,12 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction from freezegun import freeze_time from requests.models import Response from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream, SourceGoogleAnalyticsDataApi +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction + from .utils import read_incremental @@ -85,10 +86,9 @@ def test_request_body_json(patch_base_class): {"name": "browser"}, ], "keepEmptyRows": True, - "dateRanges": [{ - "startDate": request_body_params["stream_slice"]["startDate"], - "endDate": request_body_params["stream_slice"]["endDate"] - }], + "dateRanges": [ + {"startDate": request_body_params["stream_slice"]["startDate"], "endDate": request_body_params["stream_slice"]["endDate"]} + ], "returnPropertyQuota": True, "offset": str(0), "limit": "100000", @@ -269,8 +269,8 @@ def test_should_retry(patch_base_class, http_status, response_action_expected, r if response_body: json_data = response_body response_mock._content = str.encode(json.dumps(json_data)) - response_mock.headers = {'Content-Type': 'application/json'} - response_mock.encoding = 'utf-8' + response_mock.headers = {"Content-Type": "application/json"} + response_mock.encoding = "utf-8" stream = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]) assert stream.get_error_handler().interpret_response(response_mock).response_action == response_action_expected @@ -312,6 +312,7 @@ def test_stream_slices(): {"startDate": "2022-12-30", "endDate": "2023-01-01"}, ] + @freeze_time("2023-01-01 00:00:00") def test_full_refresh(): """ @@ -319,9 +320,7 @@ def test_full_refresh(): """ config = {"date_ranges_start_date": datetime.date(2022, 12, 29), "window_in_days": 1, "dimensions": ["browser", "country", "language"]} stream = GoogleAnalyticsDataApiBaseStream(authenticator=None, config=config) - full_refresh_state = { - "__ab_full_refresh_state_message": True - } + full_refresh_state = {"__ab_full_refresh_state_message": True} slices = list(stream.stream_slices(sync_mode=None, stream_state=full_refresh_state)) assert slices == [ {"startDate": "2022-12-29", "endDate": "2022-12-29"}, @@ -423,47 +422,37 @@ def test_read_incremental(requests_mock): {"property_id": 123, "yearWeek": "202202", "totalUsers": 140, "startDate": "2022-01-10", "endDate": "2022-01-10"}, ] + @pytest.mark.parametrize( "config_dimensions, expected_state", [ pytest.param(["browser", "country", "language", "date"], {"date": "20240320"}, id="test_date_no_cursor_field_dimension"), pytest.param(["browser", "country", "language"], {}, id="test_date_cursor_field_dimension"), - ] + ], ) def test_get_updated_state(config_dimensions, expected_state): config = { - "credentials": { - "auth_type": "Service", - "credentials_json": "{ \"client_email\": \"a@gmail.com\", \"client_id\": \"1234\", \"client_secret\": \"5678\", \"private_key\": \"5678\"}" - }, - "date_ranges_start_date": "2023-04-01", - "window_in_days": 30, - "property_ids": ["123"], - "custom_reports_array": [ - { - "name": "pivot_report", - "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], - "dimensions": config_dimensions, - "metrics": ["sessions"], - "pivots": [ - { - "fieldNames": ["browser"], - "limit": 5 - }, - { - "fieldNames": ["country"], - "limit": 250 - }, + "credentials": { + "auth_type": "Service", + "credentials_json": '{ "client_email": "a@gmail.com", "client_id": "1234", "client_secret": "5678", "private_key": "5678"}', + }, + "date_ranges_start_date": "2023-04-01", + "window_in_days": 30, + "property_ids": ["123"], + "custom_reports_array": [ { - "fieldNames": ["language"], - "limit": 15 + "name": "pivot_report", + "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], + "dimensions": config_dimensions, + "metrics": ["sessions"], + "pivots": [ + {"fieldNames": ["browser"], "limit": 5}, + {"fieldNames": ["country"], "limit": 250}, + {"fieldNames": ["language"], "limit": 15}, + ], + "cohortSpec": {"enabled": "false"}, } - ], - "cohortSpec": { - "enabled": "false" - } - } - ] + ], } source = SourceGoogleAnalyticsDataApi() config = source._validate_and_transform(config, report_names=set()) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py index 4d84f48a074b2..d2371e025304a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4-service-account-only/main.py @@ -10,6 +10,7 @@ from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type from airbyte_cdk.utils import AirbyteTracedException + if __name__ == "__main__": logger = init_logger("airbyte") init_uncaught_exception_handler(logger) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/main.py b/airbyte-integrations/connectors/source-google-analytics-v4/main.py index 4d84f48a074b2..d2371e025304a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/main.py @@ -10,6 +10,7 @@ from airbyte_cdk.models import AirbyteMessage, ConnectorSpecification, FailureType, Type from airbyte_cdk.utils import AirbyteTracedException + if __name__ == "__main__": logger = init_logger("airbyte") init_uncaught_exception_handler(logger) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py index 9e6d232996e60..6f0b224882f32 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, ValidationError, validator from pydantic.class_validators import root_validator + ERROR_MSG_MISSING_SEGMENT_DIMENSION = "errors: `ga:segment` is required" @@ -99,7 +100,6 @@ def explain(self, errors: List[Dict]): @dataclass class CustomReportsValidator: - custom_reports: Union[List[Dict], Dict] = Field(default_factory=list) def __post_init__(self): @@ -108,7 +108,6 @@ def __post_init__(self): self.explainer: Explainer = Explainer() def validate(self): - # local import of airbyte_cdk dependencies from airbyte_cdk.models import FailureType from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index a71786d1a3a99..5bfa7b4cf297f 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -14,6 +14,7 @@ import jwt import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -23,6 +24,7 @@ from .custom_reports_validator import CustomReportsValidator + DATA_IS_NOT_GOLDEN_MSG = "Google Analytics data is not golden. Future requests may return different data." RESULT_IS_SAMPLED_MSG = ( @@ -179,7 +181,6 @@ def raise_on_http_errors(self) -> bool: def request_body_json( self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs: Any ) -> Optional[Mapping]: - metrics = [{"expression": metric} for metric in self.metrics] dimensions = [{"name": dimension} for dimension in self.dimensions] segments = [{"segmentId": segment} for segment in self.segments] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py index 35e9d17716d8f..58cb13723d301 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pendulum import pytest + from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.sources.streams.http.auth import NoAuth diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py index 4becd6c2d323b..023e077dfd44c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_google_analytics_v4.custom_reports_validator import CustomReportsValidator +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @pytest.mark.parametrize( "custom_reports, expected", diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index a4a9f276ba149..bd6cdc1ac209e 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -8,7 +8,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode, Type from freezegun import freeze_time from source_google_analytics_v4.source import ( DATA_IS_NOT_GOLDEN_MSG, @@ -19,6 +18,9 @@ SourceGoogleAnalyticsV4, ) +from airbyte_cdk.models import SyncMode, Type + + expected_metrics_dimensions_type_map = ( {"ga:users": "INTEGER", "ga:newUsers": "INTEGER"}, {"ga:date": "STRING", "ga:country": "STRING"}, diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-directory/main.py b/airbyte-integrations/connectors/source-google-directory/main.py index fa60e31af90e0..62faf30d2f24b 100644 --- a/airbyte-integrations/connectors/source-google-directory/main.py +++ b/airbyte-integrations/connectors/source-google-directory/main.py @@ -4,5 +4,6 @@ from source_google_directory.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py index 6a74d4cd0d751..9151f37a94bce 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py @@ -17,6 +17,7 @@ from .utils import rate_limit_handling + SCOPES = ["https://www.googleapis.com/auth/admin.directory.user.readonly", "https://www.googleapis.com/auth/admin.directory.group.readonly"] diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py index 6b0c294530cd2..abe3d6fde3eda 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py @@ -7,6 +7,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-drive/main.py b/airbyte-integrations/connectors/source-google-drive/main.py index 606e4f7641e8c..53fba38d78460 100644 --- a/airbyte-integrations/connectors/source-google-drive/main.py +++ b/airbyte-integrations/connectors/source-google-drive/main.py @@ -4,5 +4,6 @@ from source_google_drive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py index 95c7572471c64..fdecc52aab916 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Union import dpath.util +from pydantic import BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from pydantic import BaseModel, Field class OAuthCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py index ac73c84b5a19e..3cf03d1759094 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -10,16 +10,18 @@ from io import IOBase from typing import Iterable, List, Optional, Set -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from google.oauth2 import credentials, service_account from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload + +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_google_drive.utils import get_folder_id from .spec import SourceGoogleDriveSpec + FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" GOOGLE_DOC_MIME_TYPE = "application/vnd.google-apps.document" EXPORTABLE_DOCUMENTS_MIME_TYPES = [ diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py index a98f2436cae49..66fce445ada74 100644 --- a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py @@ -7,11 +7,12 @@ from unittest.mock import MagicMock, call, patch import pytest +from source_google_drive.spec import ServiceAccountCredentials, SourceGoogleDriveSpec +from source_google_drive.stream_reader import GoogleDriveRemoteFile, SourceGoogleDriveStreamReader + from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode -from source_google_drive.spec import ServiceAccountCredentials, SourceGoogleDriveSpec -from source_google_drive.stream_reader import GoogleDriveRemoteFile, SourceGoogleDriveStreamReader def create_reader( @@ -19,7 +20,7 @@ def create_reader( folder_url="https://drive.google.com/drive/folders/1Z2Q3", streams=[FileBasedStreamConfig(name="test", format=JsonlFormat())], credentials=ServiceAccountCredentials(auth_type="Service", service_account_info='{"test": "abc"}'), - ) + ), ): reader = SourceGoogleDriveStreamReader() reader.config = config diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py index 8dcb7e52e2236..2740a3d88ee5d 100644 --- a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py @@ -18,11 +18,11 @@ ("https://drive.google.com/drive/my-drive", None, True), ("http://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", None, True), ("https://drive.google.com/", None, True), - ] + ], ) def test_get_folder_id(input, output, raises): if raises: with pytest.raises(ValueError): get_folder_id(input) else: - assert get_folder_id(input) == output \ No newline at end of file + assert get_folder_id(input) == output diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py b/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py index 944377055e876..20d8f3b0132a0 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/get_authentication_url.py @@ -4,6 +4,7 @@ import json + # Check https://developers.google.com/webmaster-tools/search-console-api-original/v3/ for all available scopes OAUTH_SCOPE = "https://www.googleapis.com/auth/webmasters.readonly" diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py b/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py index 675661179530e..6ceb6ae845a5c 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/get_refresh_token.py @@ -8,6 +8,7 @@ import requests + GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token" with open("credentials.json", "r") as f: diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py index 4e39115533b46..906add1a3e0ee 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "requests", ] diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-search-console/main.py b/airbyte-integrations/connectors/source-google-search-console/main.py index 845383457bb79..62b40b4e03912 100755 --- a/airbyte-integrations/connectors/source-google-search-console/main.py +++ b/airbyte-integrations/connectors/source-google-search-console/main.py @@ -4,5 +4,6 @@ from source_google_search_console.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py index 2774f64703c72..7e0886377db3e 100644 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py index 15759c39bec7b..0530faa64a014 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py @@ -6,8 +6,10 @@ from google.auth.transport.requests import Request from google.oauth2.service_account import Credentials from requests.auth import AuthBase + from source_google_search_console.exceptions import UnauthorizedServiceAccountError + DEFAULT_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"] diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index e1481d1dcaab6..c8c196c22f4cd 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -10,6 +10,7 @@ import jsonschema import pendulum import requests + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -40,6 +41,7 @@ Sites, ) + custom_reports_schema = { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index 1474a7591f7ab..76d359743d65e 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -9,10 +9,12 @@ import pendulum import requests +from requests.auth import AuthBase + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream -from requests.auth import AuthBase + BASE_URL = "https://www.googleapis.com/webmasters/v3/" ROW_LIMIT = 25000 diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py index 1a321480fc1f8..63498cf5e2f42 100644 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_google_search_console.config_migrations import MigrateCustomReports from source_google_search_console.source import SourceGoogleSearchConsole +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 3c2b6dedfbe56..72acd67241e45 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -8,8 +8,6 @@ import pytest import requests -from airbyte_cdk.models import AirbyteConnectionStatus, Status, SyncMode -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest_lazyfixture import lazy_fixture from source_google_search_console.source import SourceGoogleSearchConsole from source_google_search_console.streams import ( @@ -23,6 +21,10 @@ ) from utils import command_check +from airbyte_cdk.models import AirbyteConnectionStatus, Status, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + logger = logging.getLogger("airbyte") @@ -133,7 +135,9 @@ def test_forbidden_should_retry(requests_mock, forbidden_error_message_json): def test_bad_aggregation_type_should_retry(requests_mock, bad_aggregation_type): stream = SearchAnalyticsKeywordSiteReportBySite(None, ["https://example.com"], "2021-01-01", "2021-01-02") - requests_mock.post(f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]}) + requests_mock.post( + f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]} + ) slice = list(stream.stream_slices(None))[0] url = stream.url_base + stream.path(None, slice) requests_mock.get(url, status_code=400, json=bad_aggregation_type) diff --git a/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-sheets/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-google-sheets/main.py b/airbyte-integrations/connectors/source-google-sheets/main.py index 806ac60fbefe5..7ecf2f8cd0e9e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/main.py +++ b/airbyte-integrations/connectors/source-google-sheets/main.py @@ -4,5 +4,6 @@ from source_google_sheets.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py index 4abb69cece9e8..1664da763bd02 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py @@ -11,6 +11,7 @@ from .helpers import SCOPES, Helpers + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py index 13b7d6f9b4bc3..74da7aef6182f 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py @@ -9,14 +9,16 @@ from datetime import datetime from typing import Dict, FrozenSet, Iterable, List, Tuple -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode from google.oauth2 import credentials as client_account from google.oauth2 import service_account from googleapiclient import discovery +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode + from .models.spreadsheet import RowData, Spreadsheet from .utils import safe_name_conversion + SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly", "https://www.googleapis.com/auth/drive.readonly"] logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py index eae6e37761231..06ba0e617d123 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py @@ -8,6 +8,10 @@ import socket from typing import Any, Generator, List, Mapping, MutableMapping, Optional, Union +from apiclient import errors +from google.auth import exceptions as google_exceptions +from requests.status_codes import codes as status_codes + from airbyte_cdk.models import FailureType from airbyte_cdk.models.airbyte_protocol import ( AirbyteCatalog, @@ -24,9 +28,6 @@ from airbyte_cdk.sources.streams.checkpoint import FullRefreshCheckpointReader from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.stream_status_utils import as_airbyte_message -from apiclient import errors -from google.auth import exceptions as google_exceptions -from requests.status_codes import codes as status_codes from .client import GoogleSheetsClient from .helpers import Helpers @@ -34,6 +35,7 @@ from .models.spreadsheet_values import SpreadsheetValues from .utils import exception_description_by_status_code, safe_name_conversion + # override default socket timeout to be 10 mins instead of 60 sec. # on behalf of https://github.com/airbytehq/oncall/issues/242 DEFAULT_SOCKET_TIMEOUT: int = 600 diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py index 0e2168c4aff2f..bf3f6a8cc302d 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py @@ -8,6 +8,7 @@ import unidecode from requests.status_codes import codes as status_codes + TOKEN_PATTERN = re.compile(r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P[^a-zA-Z\d]+)") DEFAULT_SEPARATOR = "_" diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py index 7f1a4c699a8b6..6c8e392764df8 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet, SpreadsheetValues, ValueRange +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + @pytest.fixture def invalid_config(): @@ -28,10 +29,11 @@ def maker(spreadsheet_id, sheet_name): sheets=[ Sheet( data=[GridData(rowData=[RowData(values=[CellData(formattedValue="ID")])])], - properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}) + properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}), ), ], ) + return maker @@ -39,6 +41,7 @@ def maker(spreadsheet_id, sheet_name): def spreadsheet_values(): def maker(spreadsheet_id): return SpreadsheetValues(spreadsheetId=spreadsheet_id, valueRanges=[ValueRange(values=[["1"]])]) + return maker @@ -51,4 +54,5 @@ def maker(*name_schema_pairs): sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) + return maker diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py index d86d9fbaaa2bc..f54e8cffb53a8 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py @@ -6,9 +6,10 @@ from unittest.mock import patch from urllib.parse import parse_qsl, unquote, urlencode, urlunparse +from httplib2 import Response + from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.request import HttpRequest -from httplib2 import Response def parse_and_transform(parse_result_str: str): @@ -20,14 +21,16 @@ def parse_and_transform(parse_result_str: str): # Convert the ParseResult string into a dictionary components = eval(f"dict({parse_result_part})") - url = urlunparse(( - components["scheme"], - components["netloc"], - components["path"], - components["params"], - components["query"], - components["fragment"], - )) + url = urlunparse( + ( + components["scheme"], + components["netloc"], + components["path"], + components["params"], + components["query"], + components["fragment"], + ) + ) return url @@ -40,6 +43,7 @@ class CustomHttpMocker: Note: there is only support for get and post method and url matching ignoring the body but this is enough for the current test set. """ + requests_mapper: Dict = {} def post(self, request: HttpRequest, response: HttpResponse): @@ -62,9 +66,9 @@ def mock_request(self, uri, method="GET", body=None, headers=None, **kwargs): return mocked_response # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` - def __call__(self, test_func): # type: ignore + def __call__(self, test_func): # type: ignore @wraps(test_func) - def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed + def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed kwargs["http_mocker"] = self with patch("httplib2.Http.request", side_effect=self.mock_request): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py index af585b9e5fd44..9fc77d47a852e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py @@ -4,17 +4,18 @@ from airbyte_cdk.test.mock_http.request import HttpRequest + # todo: this should be picked from manifest in the future GOOGLE_SHEETS_BASE_URL = "https://sheets.googleapis.com/v4/spreadsheets" OAUTH_AUTHORIZATION_ENDPOINT = "https://oauth2.googleapis.com" + class RequestBuilder: @classmethod def get_account_endpoint(cls) -> RequestBuilder: return cls(resource="values:batchGet") - - def __init__(self, resource:str=None) -> None: + def __init__(self, resource: str = None) -> None: self._spreadsheet_id = None self._query_params = {} self._body = None diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py index 4fdcd7fe0309d..e4781f4d267d4 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import json + # this key was generated with rsa library from cryptography test_private_key = """ -----BEGIN PRIVATE KEY----- @@ -59,5 +60,5 @@ "auth_type": "Client", "client_id": "43987534895734985.apps.googleusercontent.com", "client_secret": "2347586435987643598", - "refresh_token": "1//4398574389537495437983457985437" - } + "refresh_token": "1//4398574389537495437983457985437", +} diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py index 6075f68535923..2123a2caec3b6 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py @@ -5,10 +5,14 @@ from copy import deepcopy from typing import Optional from unittest import TestCase -from unittest.mock import patch # patch("time.sleep") -from unittest.mock import Mock +from unittest.mock import ( + Mock, + patch, # patch("time.sleep") +) import pytest +from source_google_sheets import SourceGoogleSheets + from airbyte_cdk.models import Status from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder @@ -16,12 +20,12 @@ from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.utils import AirbyteTracedException -from source_google_sheets import SourceGoogleSheets from .custom_http_mocker import CustomHttpMocker as HttpMocker from .request_builder import AuthBuilder, RequestBuilder from .test_credentials import oauth_credentials, service_account_credentials, service_account_info + _SPREADSHEET_ID = "a_spreadsheet_id" _STREAM_NAME = "a_stream_name" @@ -29,15 +33,9 @@ _C_STREAM_NAME = "c_stream_name" -_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": oauth_credentials -} +_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": oauth_credentials} -_SERVICE_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": service_account_credentials -} +_SERVICE_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": service_account_credentials} class GoogleSheetSourceTest(TestCase): @@ -49,14 +47,11 @@ def setUp(self) -> None: @staticmethod def authorize(http_mocker: HttpMocker): # Authorization request with user credentials to "https://oauth2.googleapis.com" to obtain a token - http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_response", __file__)), 200) - ) + http_mocker.post(AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_response", __file__)), 200)) @staticmethod - def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=None, meta_response_code: Optional[int]=200): - """" + def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str] = None, meta_response_code: Optional[int] = 200): + """ " Mock request to https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=false&alt=json in order to obtain sheets (streams) from the spreed_sheet_id provided. e.g. from response file @@ -79,12 +74,19 @@ def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=No if streams_response_file: http_mocker.get( RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(False).with_alt("json").build(), - HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code) + HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code), ) @staticmethod - def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_response_code: int=200, stream_name: Optional[str]=_STREAM_NAME, data_initial_range_response_file: Optional[str]=None, data_initial_response_code: Optional[int]=200): - """" + def get_schema( + http_mocker: HttpMocker, + headers_response_file: str, + headers_response_code: int = 200, + stream_name: Optional[str] = _STREAM_NAME, + data_initial_range_response_file: Optional[str] = None, + data_initial_response_code: Optional[int] = 200, + ): + """ " Mock request to 'https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=true&ranges=!1:1&alt=json' to obtain headers data (keys) used for stream schema from the spreadsheet + sheet provided. For this we use range of first row in query. @@ -127,13 +129,20 @@ def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_resp ]}],}]}] """ http_mocker.get( - RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(True).with_ranges(f"{stream_name}!1:1").with_alt("json").build(), - HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code) + RequestBuilder() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_include_grid_data(True) + .with_ranges(f"{stream_name}!1:1") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code), ) @staticmethod - def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int=200, stream_name:Optional[str]=_STREAM_NAME): - """" + def get_stream_data( + http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int = 200, stream_name: Optional[str] = _STREAM_NAME + ): + """ " Mock requests to 'https://sheets.googleapis.com/v4/spreadsheets//values:batchGet?ranges=!2:202&majorDimension=ROWS&alt=json' to obtain value ranges (data) for stream from the spreadsheet + sheet provided. For this we use range [2:202(2 + range in config which default is 200)]. @@ -156,8 +165,13 @@ def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, rang """ batch_request_ranges = f"{stream_name}!2:202" http_mocker.get( - RequestBuilder.get_account_endpoint().with_spreadsheet_id(_SPREADSHEET_ID).with_ranges(batch_request_ranges).with_major_dimension("ROWS").with_alt("json").build(), - HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code) + RequestBuilder.get_account_endpoint() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_ranges(batch_request_ranges) + .with_major_dimension("ROWS") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code), ) @HttpMocker() @@ -186,15 +200,12 @@ def test_given_service_authentication_error_when_check_then_status_is_failed(sel @HttpMocker() def test_invalid_credentials_error_message_when_check(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." - ) + assert str(exc_info.value) == ("Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.") @HttpMocker() def test_invalid_link_error_message_when_check(self, http_mocker: HttpMocker) -> None: @@ -216,9 +227,7 @@ def test_check_access_expired(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_permissions", 403) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Config error: " - ) + assert str(exc_info.value) == ("Config error: ") @HttpMocker() def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) -> None: @@ -227,7 +236,10 @@ def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f'Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. ' + assert ( + connection_status.message + == f"Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " + ) @HttpMocker() def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: @@ -236,7 +248,10 @@ def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + assert ( + connection_status.message + == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + ) @HttpMocker() def test_given_grid_sheet_type_with_at_least_one_row_when_discover_then_return_stream(self, http_mocker: HttpMocker) -> None: @@ -252,9 +267,9 @@ def test_discover_return_expected_schema(self, http_mocker: HttpMocker) -> None: # When we move to manifest only is possible that DynamicSchemaLoader will identify fields like age as integers # and addresses "oneOf": [{"type": ["null", "string"]}, {"type": ["null", "integer"]}] as it has mixed data expected_schemas_properties = { - _STREAM_NAME: {'age': {'type': 'string'}, 'name': {'type': 'string'}}, - _B_STREAM_NAME: {'email': {'type': 'string'}, 'name': {'type': 'string'}}, - _C_STREAM_NAME: {'address': {'type': 'string'}} + _STREAM_NAME: {"age": {"type": "string"}, "name": {"type": "string"}}, + _B_STREAM_NAME: {"email": {"type": "string"}, "name": {"type": "string"}}, + _C_STREAM_NAME: {"address": {"type": "string"}}, } GoogleSheetSourceTest.get_streams(http_mocker, "multiple_streams_schemas_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, f"multiple_streams_schemas_{_STREAM_NAME}_range", 200) @@ -293,13 +308,12 @@ def test_discover_could_not_run_discover(self, http_mocker: HttpMocker) -> None: @HttpMocker() def test_discover_invalid_credentials_error_message(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(Exception) as exc_info: self._source.discover(Mock(), self._config) - expected_message = 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + expected_message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." assert str(exc_info.value) == expected_message @HttpMocker() @@ -307,7 +321,6 @@ def test_discover_404_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_link", 404) with pytest.raises(AirbyteTracedException) as exc_info: - self._source.discover(Mock(), self._config) expected_message = ( f"The requested Google Sheets spreadsheet with id {_SPREADSHEET_ID} does not exist." @@ -320,7 +333,6 @@ def test_discover_403_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_permissions", 403) with pytest.raises(AirbyteTracedException) as exc_info: - self._source.discover(Mock(), self._config) expected_message = ( "The authenticated Google Sheets user does not have permissions to view the " @@ -350,7 +362,15 @@ def test_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range") GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 2 @@ -367,20 +387,23 @@ def test_when_read_multiple_streams_return_records(self, http_mocker: HttpMocker GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_B_STREAM_NAME}_range_2", stream_name=_B_STREAM_NAME) GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_C_STREAM_NAME}_range_2", stream_name=_C_STREAM_NAME) - configured_catalog = (CatalogBuilder().with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME). - with_json_schema({"properties": {'age': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_B_STREAM_NAME). - with_json_schema({"properties": {'email': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME). - with_json_schema({"properties": {'address': {'type': 'string'}} - }) + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"age": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_B_STREAM_NAME) + .with_json_schema({"properties": {"email": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME).with_json_schema({"properties": {"address": {"type": "string"}}}) + ) + .build() ) - .build()) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 9 @@ -403,7 +426,15 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_2", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions_2") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 5 @@ -414,21 +445,26 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt assert output.trace_messages[1].trace.stream_status.status == AirbyteStreamStatus.RUNNING assert output.trace_messages[2].trace.stream_status.status == AirbyteStreamStatus.COMPLETE - @HttpMocker() def test_read_429_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "rate_limit_error", 429) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -437,13 +473,19 @@ def test_read_403_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "invalid_permissions", 403) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" - ) + expected_message = f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -452,14 +494,20 @@ def test_read_500_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "internal_server_error", 500) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" - ) + expected_message = "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -467,12 +515,18 @@ def test_read_empty_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_empty", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message @HttpMocker() @@ -480,10 +534,16 @@ def test_read_expected_data_on_1_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_with_unexpected_extra_sheet", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index 5a1408faa0c17..a57d024c6d0c5 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -7,6 +7,10 @@ import unittest from unittest.mock import Mock, patch +from source_google_sheets.client import GoogleSheetsClient +from source_google_sheets.helpers import Helpers +from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet + from airbyte_cdk.models.airbyte_protocol import ( AirbyteRecordMessage, AirbyteStream, @@ -15,9 +19,7 @@ DestinationSyncMode, SyncMode, ) -from source_google_sheets.client import GoogleSheetsClient -from source_google_sheets.helpers import Helpers -from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index b939910d57f7c..54a2e054d91e4 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -6,14 +6,15 @@ import pytest import requests -from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog -from airbyte_cdk.utils import AirbyteTracedException from apiclient import errors from source_google_sheets import SourceGoogleSheets from source_google_sheets.client import GoogleSheetsClient from source_google_sheets.helpers import SCOPES, Helpers from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet +from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog +from airbyte_cdk.utils import AirbyteTracedException + def set_http_error_for_google_sheets_client(mocker, resp): mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) @@ -191,7 +192,7 @@ def test_discover_invalid_credentials_error_message(mocker, invalid_config): source = SourceGoogleSheets() with pytest.raises(AirbyteTracedException) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - assert e.value.args[0] == 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + assert e.value.args[0] == "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." def test_get_credentials(invalid_config): @@ -223,12 +224,14 @@ def test_read_429_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) - expected_message = ( - "Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Rate limit has been reached. Please try later or request a higher quota for your account." assert e.value.args[0] == expected_message @@ -243,7 +246,11 @@ def test_read_403_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) assert ( @@ -265,7 +272,11 @@ def test_read_500_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) expected_message = ( @@ -303,8 +314,13 @@ def test_read_empty_sheet(invalid_config, mocker, catalog, caplog): sheet1 = "soccer_team" sheet2 = "soccer_team2" sheets = [ - Sheet(properties=SheetProperties(title=t), data=[{"test1": "12", "test2": "123"},]) - for t in [sheet1] + Sheet( + properties=SheetProperties(title=t), + data=[ + {"test1": "12", "test2": "123"}, + ], + ) + for t in [sheet1] ] mocker.patch.object( GoogleSheetsClient, @@ -328,7 +344,11 @@ def test_when_read_then_status_messages_emitted(mocker, spreadsheet, spreadsheet mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed @@ -346,7 +366,11 @@ def test_when_read_then_state_message_emitted(mocker, spreadsheet, spreadsheet_v mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed diff --git a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-google-webfonts/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-greenhouse/main.py b/airbyte-integrations/connectors/source-greenhouse/main.py index e08a14b429fdd..378686d959b49 100644 --- a/airbyte-integrations/connectors/source-greenhouse/main.py +++ b/airbyte-integrations/connectors/source-greenhouse/main.py @@ -4,5 +4,6 @@ from source_greenhouse.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py index dabad443b3662..2be3db68fbffc 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py index 605c45c1ea2fb..463a456b4aaab 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py @@ -5,9 +5,10 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.sources.streams import Stream from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer +from airbyte_cdk.sources.streams import Stream + @pytest.fixture def greenhouse_slicer(): @@ -18,4 +19,11 @@ def greenhouse_slicer(): @pytest.fixture def greenhouse_substream_slicer(): parent_stream = MagicMock(spec=Stream) - return GreenHouseSubstreamSlicer(cursor_field='cursor_field', stream_slice_field='slice_field', parent_stream=parent_stream, parent_key='parent_key', parameters={}, request_cursor_field=None) + return GreenHouseSubstreamSlicer( + cursor_field="cursor_field", + stream_slice_field="slice_field", + parent_stream=parent_stream, + parent_key="parent_key", + parameters={}, + request_cursor_field=None, + ) diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py index 48db265f477f0..27bab35ef71ee 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -6,9 +6,10 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.sources.streams import Stream from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer +from airbyte_cdk.sources.streams import Stream + def test_slicer(greenhouse_slicer): date_time = "2022-09-05T10:10:10.000000Z" @@ -53,17 +54,12 @@ def test_sub_slicer(last_record, expected, records): @pytest.mark.parametrize( "stream_state, cursor_field, expected_state", [ - ({'cursor_field_1': '2022-09-05T10:10:10.000Z'}, 'cursor_field_1', {'cursor_field_1': '2022-09-05T10:10:10.000Z'}), - ({'cursor_field_2': '2022-09-05T10:10:100000Z'}, 'cursor_field_3', {}), - ({'cursor_field_4': None}, 'cursor_field_4', {}), - ({'cursor_field_5': ''}, 'cursor_field_5', {}), + ({"cursor_field_1": "2022-09-05T10:10:10.000Z"}, "cursor_field_1", {"cursor_field_1": "2022-09-05T10:10:10.000Z"}), + ({"cursor_field_2": "2022-09-05T10:10:100000Z"}, "cursor_field_3", {}), + ({"cursor_field_4": None}, "cursor_field_4", {}), + ({"cursor_field_5": ""}, "cursor_field_5", {}), ], - ids=[ - "cursor_value_present", - "cursor_value_not_present", - "cursor_value_is_None", - "cursor_value_is_empty_string" - ] + ids=["cursor_value_present", "cursor_value_not_present", "cursor_value_is_None", "cursor_value_is_empty_string"], ) def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer = GreenHouseSlicer(cursor_field=cursor_field, parameters={}, request_cursor_field=None) @@ -71,36 +67,30 @@ def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer.set_initial_state(stream_state) assert slicer.get_stream_state() == expected_state + @pytest.mark.parametrize( "stream_state, initial_state, expected_state", [ ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'}}, - { - 'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}, - 'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'} - } + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}, "id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, ), ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T09:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}} - ), - ( - {}, - {}, - {} + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T09:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, ), + ({}, {}, {}), ], ids=[ "stream_state and initial_state have different keys", "stream_state and initial_state have overlapping keys with different values", - "stream_state and initial_state are empty" - ] + "stream_state and initial_state are empty", + ], ) def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, initial_state, expected_state): - slicer = greenhouse_substream_slicer + slicer = greenhouse_substream_slicer # Set initial state slicer._state = initial_state slicer.set_initial_state(stream_state) @@ -110,27 +100,11 @@ def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, @pytest.mark.parametrize( "first_record, second_record, expected_result", [ - ( - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ( - {'cursor_field': '2023-02-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - True - ), - ( - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - {'cursor_field': ''}, - True - ), - ( - {'cursor_field': ''}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ] + ({"cursor_field": "2023-01-01T00:00:00.000Z"}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ({"cursor_field": "2023-02-01T00:00:00.000Z"}, {"cursor_field": "2023-01-01T00:00:00.000Z"}, True), + ({"cursor_field": "2023-01-02T00:00:00.000Z"}, {"cursor_field": ""}, True), + ({"cursor_field": ""}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ], ) def test_is_greater_than_or_equal(greenhouse_substream_slicer, first_record, second_record, expected_result): slicer = greenhouse_substream_slicer diff --git a/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gridly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-gridly/main.py b/airbyte-integrations/connectors/source-gridly/main.py index 307be6500faff..48b578302a7b6 100644 --- a/airbyte-integrations/connectors/source-gridly/main.py +++ b/airbyte-integrations/connectors/source-gridly/main.py @@ -4,5 +4,6 @@ from source_gridly.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py b/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py index 05f9d4843cc56..b896e6102f946 100644 --- a/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/helpers.py @@ -5,6 +5,7 @@ from typing import Any, Dict import requests + from airbyte_cdk.models import AirbyteStream from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-gridly/source_gridly/source.py b/airbyte-integrations/connectors/source-gridly/source_gridly/source.py index 10a400672cfb4..d1c2ed8a5483d 100644 --- a/airbyte-integrations/connectors/source-gridly/source_gridly/source.py +++ b/airbyte-integrations/connectors/source-gridly/source_gridly/source.py @@ -9,6 +9,7 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.models import AirbyteCatalog from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py index 2aa8d22ce2d96..4d3ea285f4f49 100644 --- a/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gridly/unit_tests/test_source.py @@ -6,6 +6,7 @@ from source_gridly.source import SourceGridly + CONFIG = {"api_key": "IbuIBdkFjrJps6", "grid_id": "4539o52kmdjmzwp"} diff --git a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-gutendex/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hardcoded-records/main.py b/airbyte-integrations/connectors/source-hardcoded-records/main.py index ead17772f9c35..e697227784c19 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/main.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/main.py @@ -5,5 +5,6 @@ from source_hardcoded_records.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py b/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py index 862ac9470e969..73b2369c57928 100644 --- a/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py +++ b/airbyte-integrations/connectors/source-hardcoded-records/source_hardcoded_records/source.py @@ -9,6 +9,7 @@ from .streams import Customers, DummyFields, Products + DEFAULT_COUNT = 1_000 diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py index a33c09315382d..237fb7b71ce11 100644 --- a/airbyte-integrations/connectors/source-harness/main.py +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -4,5 +4,6 @@ from source_harness.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-harness/source_harness/run.py b/airbyte-integrations/connectors/source-harness/source_harness/run.py index 544daa9407a16..6a0dc730fd62d 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/run.py +++ b/airbyte-integrations/connectors/source-harness/source_harness/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_harness import SourceHarness +from airbyte_cdk.entrypoint import launch + def run(): source = SourceHarness() diff --git a/airbyte-integrations/connectors/source-harness/source_harness/source.py b/airbyte-integrations/connectors/source-harness/source_harness/source.py index a52f4c06db862..b12559ebc98af 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/source.py +++ b/airbyte-integrations/connectors/source-harness/source_harness/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py index 9e7e8054a2737..05c6d78148091 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/test_associations.py @@ -5,9 +5,10 @@ import logging import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type from source_hubspot.source import SourceHubspot +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type + @pytest.fixture def source(): diff --git a/airbyte-integrations/connectors/source-hubspot/main.py b/airbyte-integrations/connectors/source-hubspot/main.py index dc073ca21ed64..8aa42945418c0 100644 --- a/airbyte-integrations/connectors/source-hubspot/main.py +++ b/airbyte-integrations/connectors/source-hubspot/main.py @@ -4,5 +4,6 @@ from source_hubspot.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py index e73313f80e3f2..925f2c84f06fa 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py @@ -6,9 +6,10 @@ from typing import Any import requests +from requests import HTTPError + from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError class HubspotError(AirbyteTracedException): diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 7bc5eda8ce53c..195ac0335d213 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -8,12 +8,13 @@ from itertools import chain from typing import Any, Generator, List, Mapping, Optional, Tuple, Union +from requests import HTTPError + from airbyte_cdk.models import FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction -from requests import HTTPError from source_hubspot.errors import HubspotInvalidAuth from source_hubspot.streams import ( API, @@ -68,6 +69,7 @@ Workflows, ) + """ https://github.com/airbytehq/oncall/issues/3800 we use start date 2006-01-01 as date of creation of Hubspot to retrieve all data if start date was not provided diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 2cbfbdd6a2c97..4daedbd18fcb7 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -15,6 +15,8 @@ import backoff import pendulum as pendulum import requests +from requests import HTTPError, codes + from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import Source @@ -29,7 +31,6 @@ from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException -from requests import HTTPError, codes from source_hubspot.components import NewtoLegacyFieldTransformation from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout, InvalidStartDateConfigError @@ -44,6 +45,7 @@ StoreAsIs, ) + # we got this when provided API Token has incorrect format CLOUDFLARE_ORIGIN_DNS_ERROR = 530 @@ -1515,14 +1517,12 @@ def cursor_field_datetime_format(self) -> str: class ContactsFormSubmissions(ContactsAllBase, ResumableFullRefreshMixin, ABC): - records_field = "form-submissions" filter_field = "formSubmissionMode" filter_value = "all" class ContactsMergedAudit(ContactsAllBase, ResumableFullRefreshMixin, ABC): - records_field = "merge-audits" unnest_fields = ["merged_from_email", "merged_to_email"] @@ -2137,7 +2137,6 @@ def _transform(self, records: Iterable) -> Iterable: class CompaniesPropertyHistory(PropertyHistoryV3): - scopes = {"crm.objects.companies.read"} properties_scopes = {"crm.schemas.companies.read"} entity = "companies" @@ -2419,7 +2418,6 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: now = pendulum.now(tz="UTC") for parent_slice in super().stream_slices(sync_mode, cursor_field, stream_state): - object_id = parent_slice["parent"][self.object_id_field] # We require this workaround to shorten the duration of the acceptance test run. diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index deecb0b377d3a..8d10c01fde860 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -6,6 +6,7 @@ from source_hubspot.source import SourceHubspot from source_hubspot.streams import API + NUMBER_OF_PROPERTIES = 2000 @@ -81,9 +82,16 @@ def some_credentials_fixture(): def fake_properties_list(): return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] + @pytest.fixture(name="migrated_properties_list") def migrated_properties_list(): - return ["hs_v2_date_entered_prospect", "hs_v2_date_exited_prospect", "hs_v2_cumulative_time_in_prsopect", "hs_v2_some_other_property_in_prospect"] + return [ + "hs_v2_date_entered_prospect", + "hs_v2_date_exited_prospect", + "hs_v2_cumulative_time_in_prsopect", + "hs_v2_some_other_property_in_prospect", + ] + @pytest.fixture(name="api") def api(some_credentials): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py index 048142759ca2a..1927b02a17054 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py @@ -5,9 +5,7 @@ class ConfigBuilder: def __init__(self): - self._config = { - "enable_experimental_streams": True - } + self._config = {"enable_experimental_streams": True} def with_start_date(self, start_date: str): self._config["start_date"] = start_date diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py index 17ba71bebf3c1..e9695115f5a3c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py @@ -26,8 +26,7 @@ def with_refresh_token(self, refresh_token: str): def build(self) -> HttpRequest: client_id, client_secret, refresh_token = self._params["client_id"], self._params["client_secret"], self._params["refresh_token"] return HttpRequest( - url=self.URL, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + url=self.URL, body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" ) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index d0258a26500a4..614c51e8ece61 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -29,12 +29,7 @@ def with_query(self, qp): return self def build(self) -> HttpRequest: - return HttpRequest( - url=self.URL, - query_params=self._query_params, - headers=self.headers, - body=self._request_body - ) + return HttpRequest(url=self.URL, query_params=self._query_params, headers=self.headers, body=self._request_body) class CRMStreamRequestBuilder(AbstractRequestBuilder): @@ -78,14 +73,7 @@ def _archived(self): @property def _query_params(self): - return [ - self._archived, - self._associations, - self._limit, - self._after, - self._dt_range, - self._properties - ] + return [self._archived, self._associations, self._limit, self._after, self._dt_range, self._properties] def build(self): q = "&".join(filter(None, self._query_params)) @@ -96,14 +84,7 @@ def build(self): class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): @property def _query_params(self): - return [ - self._limit, - self._after, - self._dt_range, - self._archived, - self._associations, - self._properties - ] + return [self._limit, self._after, self._dt_range, self._archived, self._associations, self._properties] class OwnersArchivedStreamRequestBuilder(AbstractRequestBuilder): @@ -122,11 +103,14 @@ def _archived(self): @property def _query_params(self): - return filter(None, [ - self._limit, - self._after, - self._archived, - ]) + return filter( + None, + [ + self._limit, + self._after, + self._archived, + ], + ) def with_page_token(self, next_page_token: Dict): self._after = "&".join([f"{str(key)}={str(val)}" for key, val in next_page_token.items()]) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py index 87ba98af2b5a8..b5c2cc79e61bd 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py @@ -7,13 +7,13 @@ from airbyte_cdk.test.mock_http import HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template + _CONTACTS_FIELD = "contacts" _FORM_SUBMISSIONS_FIELD = "form-submissions" _LIST_MEMBERSHIPS_FIELD = "list-memberships" _MERGE_AUDITS_FIELD = "merge-audits" - def _get_template() -> Dict[str, Any]: return find_template("all_contacts", __file__) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py index 595a02232d439..f5bb912822f72 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -12,7 +12,7 @@ def __init__( self, template: List[Any], records_path: Optional[Union[FieldPath, NestedPath]] = None, - pagination_strategy: Optional[PaginationStrategy] = None + pagination_strategy: Optional[PaginationStrategy] = None, ): self._response = template self._records: List[RecordBuilder] = [] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index ded25153204f4..7f594d984fdf9 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -9,13 +9,4 @@ class HubspotPaginationStrategy(PaginationStrategy): NEXT_PAGE_TOKEN = {"after": "256"} def update(self, response: Dict[str, Any]) -> None: - response["paging"] = { - "next": { - "link": "link_to_the_next_page", - **self.NEXT_PAGE_TOKEN - }, - "prev": { - "before": None, - "link": None - } - } + response["paging"] = {"next": {"link": "link_to_the_next_page", **self.NEXT_PAGE_TOKEN}, "prev": {"before": None, "link": None}} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index f5d4795c37752..6b02d952e14c0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -13,7 +13,7 @@ class HubspotStreamResponseBuilder(HttpResponseBuilder): @property def pagination_strategy(self): return self._pagination_strategy - + @classmethod def for_stream(cls, stream: str): return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py index 7bcddfe5e179d..fdf05847617cd 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py @@ -3,6 +3,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -10,6 +11,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsFormSubmissionsBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _VID_OFFSET = 5331889818 @@ -32,43 +34,60 @@ def tearDown(self) -> None: def test_read_multiple_contact_pages(self) -> None: first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( - cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), - stream=self.STREAM_NAME, - sync_mode=SyncMode.full_refresh + cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh ) self._http_mocker.assert_number_of_calls(first_page_request, 2) @@ -87,56 +106,73 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py index 8fb6e598a395c..3c21d185a6502 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py @@ -4,6 +4,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.state_builder import StateBuilder from airbyte_protocol.models import SyncMode @@ -12,6 +13,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsListMembershipBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _NOW = datetime.now(timezone.utc) @@ -37,25 +39,40 @@ def test_given_pagination_when_read_then_extract_records_from_both_pages(self) - self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).with_vid_offset(str(_VID_OFFSET)).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ]), - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ] + ), + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -67,14 +84,22 @@ def test_given_timestamp_before_start_date_when_read_then_filter_out(self) -> No self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), + ] + ), + ] + ) + .build(), + ) + output = self.read_from_stream( + self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh ) - output = self.read_from_stream(self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -83,12 +108,18 @@ def test_given_state_when_read_then_filter_out(self) -> None: self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py index 18c67317aebfb..8b76e13cc6c45 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py @@ -3,6 +3,7 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -10,6 +11,7 @@ from .request_builders.streams import ContactsStreamRequestBuilder from .response_builder.contact_response_builder import AllContactsResponseBuilder, ContactBuilder, ContactsMergeAuditsBuilder + _START_TIME_BEFORE_ANY_RECORD = "1970-01-01T00:00:00Z" _VID_OFFSET = 5331889818 @@ -36,32 +38,49 @@ def test_read_multiple_contact_pages(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -82,9 +101,8 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] @@ -94,39 +112,56 @@ def test_read_from_incoming_state(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index d37005531ecf9..936f68a4fa2a0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -5,6 +5,7 @@ import freezegun import mock + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -27,22 +28,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +82,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +99,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,8 +136,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 - diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py index d07dbd20985a9..5f46b8982ae8d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py @@ -5,6 +5,7 @@ import freezegun import mock + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -27,22 +28,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +82,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +99,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,7 +136,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index d54a94ca6a9d8..eef01cbc0fc57 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import SyncMode @@ -16,6 +17,7 @@ class TestOwnersArchivedStream(HubspotTestCase): The test case contains a single test - this is just a sanity check, as the tested stream is identical to the `Owners` stream (which is covered by acceptance tests), except for a single url param. """ + SCOPES = ["crm.objects.owners.read"] CURSOR_FIELD = "updatedAt" STREAM_NAME = "owners_archived" @@ -28,10 +30,10 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -54,7 +56,7 @@ def test_given_two_pages_when_read_stream_private_token_then_return_records(self self.mock_response( http_mocker, self.request().with_page_token(self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN).build(), - self.response().build() + self.response().build(), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 7e3889a69ee39..bd22328de3ec5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -8,6 +8,7 @@ import mock import pytest import pytz + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode @@ -16,6 +17,7 @@ from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder from .response_builder.streams import HubspotStreamResponseBuilder + CRM_STREAMS = ( ("tickets_web_analytics", "tickets", "ticket", ["contacts", "deals", "companies"]), ("deals_web_analytics", "deals", "deal", ["contacts", "companies", "line_items"]), @@ -51,17 +53,11 @@ def web_analytics_request( object_type: str, start_date: Optional[str] = None, end_date: Optional[str] = None, - first_page: bool = True + first_page: bool = True, ): start_date = start_date or cls.dt_str(cls.start_date()) end_date = end_date or cls.dt_str(cls.now()) - query = { - "limit": 100, - "occurredAfter": start_date, - "occurredBefore": end_date, - "objectId": object_id, - "objectType": object_type - } + query = {"limit": 100, "occurredAfter": start_date, "occurredBefore": end_date, "objectId": object_id, "objectType": object_type} if not first_page: query.update(cls.response_builder(stream).pagination_strategy.NEXT_PAGE_TOKEN) @@ -95,10 +91,10 @@ def mock_parent_object( ): response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: @@ -136,7 +132,7 @@ def test_given_one_page_when_read_stream_oauth_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.oauth_config(), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -154,7 +150,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -172,12 +168,12 @@ def test_given_two_pages_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, with_pagination=True) + self.web_analytics_response(stream_name, with_pagination=True), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type, first_page=False), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -196,7 +192,7 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, with_pagination=True, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_parent_object( http_mocker, @@ -205,17 +201,17 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, first_page=False, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, "another_object_id", object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -236,7 +232,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()), - date_range=start_to_end + date_range=start_to_end, ) for dt_range in date_ranges: for _id in (self.OBJECT_ID, "another_object_id"): @@ -245,7 +241,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, _id, object_type, start, end), - web_analytics_response + web_analytics_response, ) config_start_dt = date_ranges[0][0] output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN, config_start_dt), stream_name, SyncMode.full_refresh) @@ -264,7 +260,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=500, body="{}") + HttpResponse(status_code=500, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -285,10 +281,7 @@ def test_given_500_then_200_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - [ - HttpResponse(status_code=500, body="{}"), - self.web_analytics_response(stream_name) - ] + [HttpResponse(status_code=500, body="{}"), self.web_analytics_response(stream_name)], ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -299,12 +292,7 @@ def test_given_500_then_200_when_read_then_return_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_missing_scopes_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) @@ -313,12 +301,7 @@ def test_given_missing_scopes_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_unauthorized_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -328,7 +311,7 @@ def test_given_unauthorized_error_when_read_then_hault( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}") + HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -339,12 +322,7 @@ def test_given_unauthorized_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_transformed_records( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -354,7 +332,7 @@ def test_given_one_page_when_read_then_get_transformed_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) record = output.records[0].record.data @@ -365,12 +343,7 @@ def test_given_one_page_when_read_then_get_transformed_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_no_records_filtered( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): # validate that no filter is applied on the record set received from the API response self.mock_custom_objects(http_mocker) @@ -381,7 +354,7 @@ def test_given_one_page_when_read_then_get_no_records_filtered( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))) + self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -399,11 +372,9 @@ def test_given_incremental_sync_when_read_then_state_message_produced_and_state_ self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) - ) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental) assert len(output.state_messages) == 1 cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) @@ -423,15 +394,15 @@ def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) another_object_id = "another_object_id" current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}) - ) + stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -453,14 +424,14 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}) - ) + stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -493,24 +464,23 @@ def mock_parent_object( date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: response_builder = response_builder.with_pagination() start, end = date_range - request_builder = IncrementalCRMStreamRequestBuilder().for_entity( - object_type - ).with_associations( - associations - ).with_dt_range( - ("startTimestamp", cls.dt_conversion(start)), - ("endTimestamp", cls.dt_conversion(end)) - ).with_properties(properties) + request_builder = ( + IncrementalCRMStreamRequestBuilder() + .for_entity(object_type) + .with_associations(associations) + .with_dt_range(("startTimestamp", cls.dt_conversion(start)), ("endTimestamp", cls.dt_conversion(end))) + .with_properties(properties) + ) if not first_page: request_builder = request_builder.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) @@ -533,12 +503,8 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_two_pages_when_read_then_return_records( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_two_pages_when_read_then_return_records( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_two_pages_when_read_then_return_records(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_two_pages_when_read_then_return_records(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( @@ -573,12 +539,8 @@ def test_given_missing_scopes_error_when_read_then_hault( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_unauthorized_error_when_read_then_hault( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_unauthorized_error_when_read_then_hault( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_unauthorized_error_when_read_then_hault(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_unauthorized_error_when_read_then_hault(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_one_page_when_read_then_get_transformed_records( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py index b57f773270f99..e83609da66b8c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py @@ -25,32 +25,20 @@ "hs_v2_date_exited_prospect": {"type": ["null", "string"]}, "hs_date_exited_prospect": {"type": ["null", "string"]}, "hs_v2_some_other_field": {"type": ["null", "string"]}, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "hs_v2_date_entered_poetry": 1827 - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827}, { "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "properties": { - "hs_v2_date_entered_poetry": 1827 - } - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "properties": {"hs_v2_date_entered_poetry": 1827}}, { "name": "Edgar Allen Poe", "age": 215, @@ -58,8 +46,8 @@ "properties": { "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } - } + }, + }, ), ( { @@ -71,19 +59,15 @@ "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", - } + }, ), ( - { - "name": "Edgar Allen Poe", - "hs_v2_date_entered_poetry": 1827, - "hs_date_entered_poetry": 9999 - }, + {"name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999}, { "name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999, - } + }, ), ], ids=[ @@ -91,7 +75,7 @@ "Transforms record w/ flat properties", "Transform record w/ nested properties", "Does not transform record w/o need to transformation", - "Does not overwrite value for legacy field if legacy field exists" + "Does not overwrite value for legacy field if legacy field exists", ], ) def test_new_to_legacy_field_transformation(input, expected): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py index a5793b49b9576..454791f222025 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_field_type_converting.py @@ -37,7 +37,6 @@ def test_field_type_format_converting(field_type, expected): ], ) def test_bad_field_type_converting(field_type, expected, caplog, capsys): - assert Stream._get_field_props(field_type=field_type) == expected logs = caplog.records diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index b3c7e71e0d7fc..2033d84ab4338 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -12,14 +12,16 @@ import mock import pendulum import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type from source_hubspot.errors import HubspotRateLimited, InvalidStartDateConfigError from source_hubspot.helpers import APIv3Property from source_hubspot.source import SourceHubspot from source_hubspot.streams import API, Companies, Deals, Engagements, MarketingEmails, Products, Stream +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type + from .utils import read_full_refresh, read_incremental + NUMBER_OF_PROPERTIES = 2000 logger = logging.getLogger("test_client") @@ -83,7 +85,6 @@ def test_check_connection_invalid_start_date_exception(config_invalid_date): @mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams") def test_streams(requests_mock, config): - streams = SourceHubspot().streams(config) assert len(streams) == 35 @@ -91,7 +92,6 @@ def test_streams(requests_mock, config): @mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams") def test_streams_incremental(requests_mock, config_experimental): - streams = SourceHubspot().streams(config_experimental) assert len(streams) == 47 @@ -141,7 +141,7 @@ def test_cast_datetime(common_params, caplog): # if you find some diff locally try using "Ex: argument of type 'DateTime' is not iterable in the message". There could be a # difference in local environment when pendulum.parsing.__init__.py importing parse_iso8601. Anyway below is working fine # in container for now and I am not sure if this diff was just a problem with my setup. - "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object" + "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object", }, } assert expected_warning_message["log"]["message"] in caplog.text @@ -488,6 +488,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req assert len(records) == 11000 assert test_stream.state["updatedAt"] == test_stream._init_sync.to_iso8601_string() + def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common_params, fake_properties_list): """ If there are more than 10,000 records that would be returned by the Hubspot search endpoint, @@ -500,7 +501,7 @@ def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common test_stream.associations = [] def random_date(start, end): - return pendulum.from_timestamp(random.randint(start, end)/1000).to_iso8601_string() + return pendulum.from_timestamp(random.randint(start, end) / 1000).to_iso8601_string() after = 0 @@ -518,17 +519,13 @@ def custom_callback(request, context): id = int(filters[2].get("value", 0)) next = int(after) + 100 results = [ - { - "id": f"{y + id}", - "updatedAt": random_date(min_time, max_time), - "after": after - } for y in range(int(after) + 1, next + 1) + {"id": f"{y + id}", "updatedAt": random_date(min_time, max_time), "after": after} for y in range(int(after) + 1, next + 1) ] context.status_code = 200 - if ((id + next) < 11000): + if (id + next) < 11000: return {"results": results, "paging": {"next": {"after": f"{next}"}}} else: - return {"results": results, "paging": {}} # Last page + return {"results": results, "paging": {}} # Last page properties_response = [ { @@ -787,6 +784,7 @@ def test_get_granted_scopes(requests_mock, mocker): assert expected_scopes == actual_scopes + def test_get_granted_scopes_retry(requests_mock, mocker): authenticator = mocker.Mock() expected_token = "the-token" diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py index 86534f61540d8..632ed4cf3554e 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py @@ -5,6 +5,7 @@ import pytest from source_hubspot.helpers import APIv1Property, APIv3Property + lorem_ipsum = """Lorem ipsum dolor sit amet, consectetur adipiscing elit""" lorem_ipsum = lorem_ipsum.lower().replace(",", "") diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 23ba97a303e70..064f98512101c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -7,7 +7,6 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode from source_hubspot.streams import ( Campaigns, Companies, @@ -44,6 +43,8 @@ Workflows, ) +from airbyte_cdk.models import SyncMode + from .utils import read_full_refresh, read_incremental @@ -169,9 +170,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para contact_lists_v1_response = [ { "json": { - "contacts": [ - {"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]} - ] + "contacts": [{"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]}] }, "status_code": 200, } @@ -193,6 +192,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para records = read_full_refresh(stream) assert records + @pytest.mark.parametrize( "stream, endpoint, cursor_value", [ @@ -203,10 +203,12 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para ids=[ "Contacts stream with v2 field transformations", "Deals stream with v2 field transformations", - "DealsArchived stream with v2 field transformations" - ] + "DealsArchived stream with v2 field transformations", + ], ) -def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list): +def test_stream_read_with_legacy_field_transformation( + stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list +): stream = stream(**common_params) responses = [ { @@ -221,7 +223,7 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", "hs_v2_cumulative_time_in_prsopect": "1 month", "hs_v2_some_other_property_in_prospect": "Great property", - } + }, } | cursor_value ], @@ -246,44 +248,40 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v records = read_full_refresh(stream) assert records expected_record = { - "id": "test_id", - "created": "2022-02-25T16:43:11Z", - "properties": { - "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "hs_v2_latest_time_in_prospect": "1 month", - "hs_v2_cumulative_time_in_prsopect": "1 month", - "hs_v2_some_other_property_in_prospect": "Great property", - "hs_time_in_prospect": "1 month", - "hs_date_exited_prospect": "2024-02-01T00:00:00Z", - }, - "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "properties_hs_v2_latest_time_in_prospect": "1 month", - "properties_hs_v2_cumulative_time_in_prsopect": "1 month", - "properties_hs_v2_some_other_property_in_prospect": "Great property", - "properties_hs_time_in_prospect": "1 month", - "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", - } | cursor_value + "id": "test_id", + "created": "2022-02-25T16:43:11Z", + "properties": { + "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "hs_v2_latest_time_in_prospect": "1 month", + "hs_v2_cumulative_time_in_prsopect": "1 month", + "hs_v2_some_other_property_in_prospect": "Great property", + "hs_time_in_prospect": "1 month", + "hs_date_exited_prospect": "2024-02-01T00:00:00Z", + }, + "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "properties_hs_v2_latest_time_in_prospect": "1 month", + "properties_hs_v2_cumulative_time_in_prsopect": "1 month", + "properties_hs_v2_some_other_property_in_prospect": "Great property", + "properties_hs_time_in_prospect": "1 month", + "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", + } | cursor_value if isinstance(stream, Contacts): expected_record = expected_record | {"properties_hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} expected_record["properties"] = expected_record["properties"] | {"hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} else: - expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z" } - expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z" } + expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z"} + expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z"} assert records[0] == expected_record - @pytest.mark.parametrize("sync_mode", [SyncMode.full_refresh, SyncMode.incremental]) -def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): +def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): stream = DealSplits(**common_params) stream_state = { "type": "STREAM", - "stream": { - "stream_descriptor": { "name": "deal_splits" }, - "stream_state": { "updatedAt": "2021-01-01T00:00:00.000000Z" } - } + "stream": {"stream_descriptor": {"name": "deal_splits"}, "stream_state": {"updatedAt": "2021-01-01T00:00:00.000000Z"}}, } cursor_value = {"updatedAt": "2022-02-25T16:43:11Z"} responses = [ @@ -583,7 +581,7 @@ def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(req def test_get_custom_objects_metadata_success(requests_mock, custom_object_schema, expected_custom_object_json_schema, api): requests_mock.register_uri("GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) - for (entity, fully_qualified_name, schema, custom_properties) in api.get_custom_objects_metadata(): + for entity, fully_qualified_name, schema, custom_properties in api.get_custom_objects_metadata(): assert entity == "animals" assert fully_qualified_name == "p19936848_Animal" assert schema == expected_custom_object_json_schema diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py index 9bff520cf87e2..942f0428b3f55 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/conftest.py @@ -6,6 +6,7 @@ import json import pytest + from airbyte_cdk.models import ConfiguredAirbyteCatalog diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py index 6687d9e16f0c4..1703f485ba35b 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py @@ -7,9 +7,10 @@ import pendulum import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, ConfiguredAirbyteCatalog, Type from source_instagram.source import SourceInstagram +from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, ConfiguredAirbyteCatalog, Type + @pytest.fixture(name="state") def state_fixture() -> MutableMapping[str, Any]: @@ -35,12 +36,16 @@ def test_incremental_streams(self, configured_catalog, config, state): assert states, "insights should produce states" for state_msg in states: - stream_name, stream_state, state_keys_count = (state_msg.state.stream.stream_descriptor.name, - state_msg.state.stream.stream_state, - len(state_msg.state.stream.stream_state.dict())) + stream_name, stream_state, state_keys_count = ( + state_msg.state.stream.stream_descriptor.name, + state_msg.state.stream.stream_state, + len(state_msg.state.stream.stream_state.dict()), + ) assert stream_name == "user_insights", f"each state message should reference 'user_insights' stream, got {stream_name} instead" - assert isinstance(stream_state, AirbyteStateBlob), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" + assert isinstance( + stream_state, AirbyteStateBlob + ), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" assert state_keys_count == 2, f"Stream state should contain 2 partition keys, got {state_keys_count} instead" @staticmethod diff --git a/airbyte-integrations/connectors/source-instagram/main.py b/airbyte-integrations/connectors/source-instagram/main.py index 0a871930a0156..7baf96d19171a 100644 --- a/airbyte-integrations/connectors/source-instagram/main.py +++ b/airbyte-integrations/connectors/source-instagram/main.py @@ -4,5 +4,6 @@ from source_instagram.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index 16426efc1f9ab..f9da2aa0df7df 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -8,15 +8,17 @@ import backoff import pendulum -from airbyte_cdk.entrypoint import logger from cached_property import cached_property from facebook_business import FacebookAdsApi from facebook_business.adobjects import user as fb_user from facebook_business.adobjects.iguser import IGUser from facebook_business.adobjects.page import Page from facebook_business.exceptions import FacebookRequestError + +from airbyte_cdk.entrypoint import logger from source_instagram.common import InstagramAPIException, retry_pattern + backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5, max_time=600) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py index 98efba83b47b4..aad54c0e13119 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py @@ -11,6 +11,7 @@ from facebook_business.exceptions import FacebookRequestError from requests.status_codes import codes as status_codes + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py index 74ee90ff8cf8c..98feb5e581257 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/components.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/components.py @@ -5,6 +5,7 @@ from typing import Any, Dict, MutableMapping, Optional import requests + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.exponential_backoff_strategy import ( ExponentialBackoffStrategy, @@ -17,6 +18,7 @@ from .common import remove_params_from_url + GRAPH_URL = resolve_manifest(source=SourceInstagram()).record.data["manifest"]["definitions"]["base_requester"]["url_base"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index fe4a35309b95f..33f9bcdb6d7b2 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -4,11 +4,13 @@ from typing import Any, List, Mapping, Tuple import pendulum + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.core import Stream from source_instagram.api import InstagramAPI from source_instagram.streams import UserInsights + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index a01fa00b8056d..397436a31bf03 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -9,10 +9,11 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum +from cached_property import cached_property + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from cached_property import cached_property from source_instagram.api import InstagramAPI from .common import remove_params_from_url diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index 92257eaab9e16..b290127f0de52 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -8,6 +8,7 @@ from pytest import fixture from source_instagram.api import InstagramAPI as API + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -57,12 +58,11 @@ def fb_account_response_fixture(account_id, some_config, requests_mock): "access_token": "access_token", "category": "Software company", "id": f"act_{account_id}", - "paging": {"cursors": { - "before": "cursor", - "after": "cursor"}}, + "paging": {"cursors": {"before": "cursor", "after": "cursor"}}, "summary": {"total_count": 1}, - "status_code": 200 - }] + "status_code": 200, + } + ] } } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py index b26f69f8a237e..ef3ea86c51dbb 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/config.py @@ -7,6 +7,7 @@ from typing import Any, List, MutableMapping + ACCESS_TOKEN = "test_access_token" ACCOUNT_ID = "111111111111111" diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py index 670d3c4c777ef..f77b436893b4a 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/pagination.py @@ -6,6 +6,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "QVFIUlhOX3Rnbm5Y" diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py index 7ac44159c282c..d10e4840f9bba 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py @@ -5,12 +5,14 @@ from typing import List, Optional, Union +from source_instagram.source import SourceInstagram + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.test.mock_http.request import HttpRequest -from source_instagram.source import SourceInstagram from .config import ACCOUNTS_FIELDS + GRAPH_URL = resolve_manifest(source=SourceInstagram()).record.data["manifest"]["definitions"]["base_requester"]["url_base"] @@ -94,4 +96,3 @@ def build(self) -> HttpRequest: def _item_path(self) -> str: path_for_resource = "/" if self._item_id_is_sub_path else "" return f"{self._item_id}{path_for_resource}" if self._item_id else "" - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py index 4a1746e422aa0..67d14cad0a99e 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py @@ -23,20 +23,7 @@ def build_response( def get_account_response() -> HttpResponse: response = { - "data": [ - { - "id": PAGE_ID, - "name": "AccountName", - "instagram_business_account": { - "id": BUSINESS_ACCOUNT_ID - } - } - ], - "paging": { - "cursors": { - "before": "before_token", - "after": "after_token" - } - } - } + "data": [{"id": PAGE_ID, "name": "AccountName", "instagram_business_account": {"id": BUSINESS_ACCOUNT_ID}}], + "paging": {"cursors": {"before": "before_token", "after": "after_token"}}, + } return build_response(body=response, status_code=HTTPStatus.OK) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py index 850356c62f494..218692c282f4b 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py @@ -21,10 +21,8 @@ from .response_builder import get_account_response from .utils import config, read_output -_FIELDS = [ - "id", - "instagram_business_account" -] + +_FIELDS = ["id", "instagram_business_account"] _STREAM_NAME = "Api" @@ -46,7 +44,6 @@ def _record() -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -70,10 +67,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_accounts_with_no_instagram_business_account_field(self, http_mocker: HttpMocker) -> None: test = "not_instagram_business_account" mocked_response = json.dumps(find_template(f"api_for_{test}", __file__)) - http_mocker.get( - get_account_request().build(), - HttpResponse(mocked_response, 200) - ) + http_mocker.get(get_account_request().build(), HttpResponse(mocked_response, 200)) original_records = json.loads(mocked_response) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py index 429b75faa4254..9319563fb463c 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py @@ -23,13 +23,15 @@ from .response_builder import get_account_response from .utils import config, read_output + _FIELDS = [ "caption", "comments_count", "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -38,44 +40,21 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_CHILDREN_FIELDS = [ - "id", - "ig_id", - "media_type", - "media_url", - "owner", - "permalink", - "shortcode", - "thumbnail_url", - "timestamp", - "username" - ] - -_CHILDREN_IDS = [ - "07608776690540123", - "52896800415362123", - "39559889460059123", - "17359925580923123" - ] +_CHILDREN_FIELDS = ["id", "ig_id", "media_type", "media_url", "owner", "permalink", "shortcode", "thumbnail_url", "timestamp", "username"] + +_CHILDREN_IDS = ["07608776690540123", "52896800415362123", "39559889460059123", "17359925580923123"] _STREAM_NAME = "media" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(_FIELDS) -def _get_children_request(media_id:str) -> RequestBuilder: - return( - RequestBuilder.get_media_children_endpoint(item_id=media_id) - .with_fields(_CHILDREN_FIELDS) - ) +def _get_children_request(media_id: str) -> RequestBuilder: + return RequestBuilder.get_media_children_endpoint(item_id=media_id).with_fields(_CHILDREN_FIELDS) def _get_response() -> HttpResponseBuilder: @@ -93,8 +72,8 @@ def _record() -> RecordBuilder: record_id_path=FieldPath("id"), ) -class TestFullRefresh(TestCase): +class TestFullRefresh(TestCase): @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -162,15 +141,12 @@ def test_given_one_page_has_children_field(self, http_mocker: HttpMocker) -> Non get_account_request().build(), get_account_response(), ) - http_mocker.get( - _get_request().build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) - ) + http_mocker.get(_get_request().build(), HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200)) for children_id in _CHILDREN_IDS: http_mocker.get( _get_children_request(children_id).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py index 11902699b40b7..bc95b4cbc934a 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py @@ -6,6 +6,7 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( @@ -24,13 +25,15 @@ from .response_builder import get_account_response from .utils import config, read_output + PARENT_FIELDS = [ "caption", "comments_count", "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -39,9 +42,9 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_PARENT_STREAM_NAME = 'media' +_PARENT_STREAM_NAME = "media" _STREAM_NAME = "media_insights" @@ -54,28 +57,38 @@ MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS = "35076616084176125" MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "35076616084176126" -REELS = 'reels' -VIDEO_FEED = 'video_feed' -VIDEO = 'video' -CAROUSEL_ALBUM = 'carousel_album' -GENERAL_MEDIA = 'general_media' -ERROR_POSTED_BEFORE_BUSINESS = 'error_posted_before_business' -ERROR_WITH_WRONG_PERMISSIONS = 'error_with_wrong_permissions' -ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = 'error_with_wrong_permissions_code_10' +REELS = "reels" +VIDEO_FEED = "video_feed" +VIDEO = "video" +CAROUSEL_ALBUM = "carousel_album" +GENERAL_MEDIA = "general_media" +ERROR_POSTED_BEFORE_BUSINESS = "error_posted_before_business" +ERROR_WITH_WRONG_PERMISSIONS = "error_with_wrong_permissions" +ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "error_with_wrong_permissions_code_10" _MEDIA_IDS = { REELS: MEDIA_ID_REELS, VIDEO_FEED: MEDIA_ID_VIDEO_FEED, VIDEO: MEDIA_ID_VIDEO, CAROUSEL_ALBUM: MEDIA_ID_CAROUSEL_ALBUM, - GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA + GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA, } METRICS_GENERAL_MEDIA = ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"] _METRICS = { - MEDIA_ID_REELS: ["comments", "ig_reels_avg_watch_time", "ig_reels_video_view_total_time", "likes", "plays", "reach", "saved", "shares", - "ig_reels_aggregated_all_plays_count", "clips_replays_count"], + MEDIA_ID_REELS: [ + "comments", + "ig_reels_avg_watch_time", + "ig_reels_video_view_total_time", + "likes", + "plays", + "reach", + "saved", + "shares", + "ig_reels_aggregated_all_plays_count", + "clips_replays_count", + ], MEDIA_ID_VIDEO_FEED: ["impressions", "reach", "saved", "video_views"], MEDIA_ID_VIDEO: ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"], MEDIA_ID_CAROUSEL_ALBUM: ["impressions", "reach", "saved", "video_views", "shares", "follows", "profile_visits"], @@ -83,29 +96,22 @@ # Reusing general media metrics for error scenarios MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS: METRICS_GENERAL_MEDIA, MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS: METRICS_GENERAL_MEDIA, - MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA + MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA, } def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -114,15 +120,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -131,7 +135,6 @@ def _record(stream_name: str, test: str = None) -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -150,12 +153,14 @@ def test_instagram_insights_for_reels(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_REELS, metric=_METRICS[MEDIA_ID_REELS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -175,12 +180,14 @@ def test_instagram_insights_for_video_feed(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO_FEED, metric=_METRICS[MEDIA_ID_VIDEO_FEED]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -200,12 +207,14 @@ def test_instagram_insights_for_video(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO, metric=_METRICS[MEDIA_ID_VIDEO]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -225,12 +234,14 @@ def test_instagram_insights_carousel_album(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_CAROUSEL_ALBUM, metric=_METRICS[MEDIA_ID_CAROUSEL_ALBUM]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -250,12 +261,14 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -266,7 +279,6 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_insights_error_posted_before_business(self, http_mocker: HttpMocker) -> None: test = ERROR_POSTED_BEFORE_BUSINESS @@ -275,18 +287,19 @@ def test_instagram_insights_error_posted_before_business(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -305,18 +318,19 @@ def test_instagram_insights_error_with_wrong_permissions(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -336,18 +350,19 @@ def test_instagram_insights_error_with_wrong_permissions_code_10(self, http_mock get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py index abc137b78c719..ac8ec96a3e1c7 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py @@ -22,6 +22,7 @@ from .response_builder import get_account_response from .utils import config, read_output + FIELDS = [ "caption", "id", @@ -35,18 +36,14 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] _STREAM_NAME = "stories" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(FIELDS) def _get_response() -> HttpResponseBuilder: @@ -66,7 +63,6 @@ def _record() -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py index 1e9e3f0f47aa9..5e71182a23499 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py @@ -6,6 +6,7 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import ( @@ -24,6 +25,7 @@ from .response_builder import get_account_response from .utils import config, read_output + PARENT_FIELDS = [ "caption", "id", @@ -37,40 +39,32 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] -_PARENT_STREAM_NAME = 'stories' +_PARENT_STREAM_NAME = "stories" _STREAM_NAME = "story_insights" STORIES_ID = "3874523487643" STORIES_ID_ERROR_CODE_10 = "3874523487644" -HAPPY_PATH = 'story_insights_happy_path' -ERROR_10 = 'story_insights_error_code_10' +HAPPY_PATH = "story_insights_happy_path" +ERROR_10 = "story_insights_error_code_10" _METRICS = ["impressions", "reach", "replies", "follows", "profile_visits", "shares", "total_interactions"] - def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -79,15 +73,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -96,7 +88,6 @@ def _record(stream_name: str, test: str = None) -> RecordBuilder: class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -117,12 +108,14 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: # Mocking parent stream http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -133,7 +126,6 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: for metric in _METRICS: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocker) -> None: test = ERROR_10 @@ -143,18 +135,17 @@ def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocke ) # Mocking parent stream http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) # Good response http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200), ) # error 10 http_mocker.get( _get_child_request(media_id=STORIES_ID_ERROR_CODE_10, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py index a9a5d717fc0f1..48dc09b91d6b7 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py @@ -21,6 +21,7 @@ from .response_builder import get_account_response from .utils import config, read_output + _CURSOR_FIELD = "id" _STREAM_NAME = "user_lifetime_insights" @@ -29,28 +30,30 @@ def _get_request() -> RequestBuilder: return ( RequestBuilder.get_user_lifetime_insights_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_custom_param("metric", "follower_demographics").with_custom_param("period", "lifetime").with_custom_param("metric_type", "total_value").with_limit(100) + .with_custom_param("metric", "follower_demographics") + .with_custom_param("period", "lifetime") + .with_custom_param("metric_type", "total_value") + .with_limit(100) ) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), record_cursor_path=FieldPath(_CURSOR_FIELD), ) class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -76,4 +79,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) # each breakdown should produce a record assert len(output.records) == 3 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py index 2e50aae606f2d..97c3519006faf 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py @@ -21,6 +21,7 @@ from .response_builder import get_account_response from .utils import config, read_output + _FIELDS = [ "id", "biography", @@ -31,36 +32,32 @@ "name", "profile_picture_url", "username", - "website" + "website", ] _STREAM_NAME = "users" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_fields(_FIELDS) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), ) class TestFullRefresh(TestCase): - @staticmethod def _read(config_: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: return read_output( @@ -83,4 +80,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) assert len(output.records) == 1 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py index 5be5aef5c8bb2..dd58fde5f31d4 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/utils.py @@ -5,10 +5,11 @@ from typing import List, Optional +from source_instagram import SourceInstagram + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode -from source_instagram import SourceInstagram from .config import ConfigBuilder diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py index ecf06f7b64e59..4ccb16aade932 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py @@ -1,77 +1,53 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. children_record = { - "children": { - "data": [ - { - "id": "7608776690540" - }, - { - "id": "2896800415362" - }, - { - "id": "9559889460059" - }, - { - "id": "7359925580923" - } - ] - } + "children": {"data": [{"id": "7608776690540"}, {"id": "2896800415362"}, {"id": "9559889460059"}, {"id": "7359925580923"}]} } expected_children_transformed = { - "children": - [ - { + "children": [ + { "id": "7608776690540", "ig_id": "2521545917836833225", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=tTUSyCXbN40Q7kNvgF-k3H6&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYAe4PKmuen4Dryt4sMEbfvrW2_eANbY1AEdl7gHG0a3mw&oe=66701C8F", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "2896800415362", "ig_id": "2521545917736276706", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=100&ccb=1-7&_nc_sid=18de74&_nc_ohc=9qJ5-fOc9lcQ7kNvgFirW6U&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBSuGRqMEzjxHri30L5NDs0irt7_7h-arKTYS8inrL56g&oe=66702E9A", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "9559889460059", "ig_id": "2521545917845406325", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=110&ccb=1-7&_nc_sid=18de74&_nc_ohc=QOtZzdxFjusQ7kNvgEqsuc2&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBPBGVS3NYW-h8oLwam_rWub-mE-9MLGc1EDVHtLJ2DBQ&oe=66702DE1", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "7359925580923", "ig_id": "2521545555591565193", "media_type": "VIDEO", "media_url": "https://fake_url?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNhcm91c2VsX2l0ZW0udW5rbm93bi1DMy40ODAuZGFzaF9iYXNlbGluZV8xX3YxIn0&_nc_ht=fakecontent&_nc_cat=108&vs=863753484982045_2117350142", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "thumbnail_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=pJkRskDC80UQ7kNvgFn3i4H&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBKK27CU9dvjiqPi9a4JKUIICp26HZ074-vgz0OVKFkbw&oe=66702104", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - } - ] + "username": "username", + }, + ] } clear_url_record = { @@ -85,293 +61,66 @@ } breakdowns_record = { - "name": "follower_demographics", - "period": "lifetime", - "title": "Follower demographics", - "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", - "total_value": { - "breakdowns": [ + "name": "follower_demographics", + "period": "lifetime", + "title": "Follower demographics", + "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", + "total_value": { + "breakdowns": [ { - "dimension_keys": [ - "city" - ], - "results": [ - { - "dimension_values": [ - "London, England" - ], - "value": 263 - }, - { - "dimension_values": [ - "Sydney, New South Wales" - ], - "value": 467 - }, - { - "dimension_values": [ - "Algiers, Algiers Province" - ], - "value": 58 - }, - { - "dimension_values": [ - "Casablanca, Grand Casablanca" - ], - "value": 71 - }, - { - "dimension_values": [ - "São Paulo, São Paulo (state)" - ], - "value": 139 - }, - { - "dimension_values": [ - "Rio de Janeiro, Rio de Janeiro (state)" - ], - "value": 44 - }, - { - "dimension_values": [ - "Perth, Western Australia" - ], - "value": 180 - }, - { - "dimension_values": [ - "Berlin, Berlin" - ], - "value": 47 - }, - { - "dimension_values": [ - "Kolkata, West Bengal" - ], - "value": 85 - }, - { - "dimension_values": [ - "Phoenix, Arizona" - ], - "value": 39 - }, - { - "dimension_values": [ - "Lagos, Lagos State" - ], - "value": 40 - }, - { - "dimension_values": [ - "Dublin, Dublin" - ], - "value": 65 - }, - { - "dimension_values": [ - "Pune, Maharashtra" - ], - "value": 72 - }, - { - "dimension_values": [ - "Wollongong, New South Wales" - ], - "value": 43 - }, - { - "dimension_values": [ - "Christchurch, Canterbury" - ], - "value": 42 - }, - { - "dimension_values": [ - "Jakarta, Jakarta" - ], - "value": 46 - }, - { - "dimension_values": [ - "Pretoria, Gauteng" - ], - "value": 54 - }, - { - "dimension_values": [ - "Buenos Aires, Ciudad Autónoma de Buenos Aires" - ], - "value": 41 - }, - { - "dimension_values": [ - "Gold Coast, Queensland" - ], - "value": 98 - }, - { - "dimension_values": [ - "Sunshine Coast, Queensland" - ], - "value": 37 - }, - { - "dimension_values": [ - "Melbourne, Victoria" - ], - "value": 338 - }, - { - "dimension_values": [ - "Gurugram, Haryana" - ], - "value": 52 - }, - { - "dimension_values": [ - "Delhi, Delhi" - ], - "value": 194 - }, - { - "dimension_values": [ - "Los Angeles, California" - ], - "value": 66 - }, - { - "dimension_values": [ - "Madrid, Comunidad de Madrid" - ], - "value": 65 - }, - { - "dimension_values": [ - "Lahore, Punjab" - ], - "value": 41 - }, - { - "dimension_values": [ - "Brisbane, Queensland" - ], - "value": 160 - }, - { - "dimension_values": [ - "Adelaide, South Australia" - ], - "value": 93 - }, - { - "dimension_values": [ - "Canberra, Australian Capital Territory" - ], - "value": 45 - }, - { - "dimension_values": [ - "Lima, Lima Region" - ], - "value": 43 - }, - { - "dimension_values": [ - "Istanbul, Istanbul Province" - ], - "value": 57 - }, - { - "dimension_values": [ - "Toronto, Ontario" - ], - "value": 40 - }, - { - "dimension_values": [ - "Chennai, Tamil Nadu" - ], - "value": 82 - }, - { - "dimension_values": [ - "Mexico City, Distrito Federal" - ], - "value": 66 - }, - { - "dimension_values": [ - "Auckland, Auckland Region" - ], - "value": 98 - }, - { - "dimension_values": [ - "Cape Town, Western Cape" - ], - "value": 172 - }, - { - "dimension_values": [ - "New York, New York" - ], - "value": 139 - }, - { - "dimension_values": [ - "Cairo, Cairo Governorate" - ], - "value": 45 - }, - { - "dimension_values": [ - "Dubai, Dubai" - ], - "value": 57 - }, - { - "dimension_values": [ - "Santiago, Santiago Metropolitan Region" - ], - "value": 73 - }, - { - "dimension_values": [ - "Mumbai, Maharashtra" - ], - "value": 195 - }, - { - "dimension_values": [ - "Bangalore, Karnataka" - ], - "value": 195 - }, - { - "dimension_values": [ - "Nairobi, Nairobi" - ], - "value": 50 - }, - { - "dimension_values": [ - "Johannesburg, Gauteng" - ], - "value": 50 - }, - { - "dimension_values": [ - "Hyderabad, Telangana" - ], - "value": 49 - } - ] + "dimension_keys": ["city"], + "results": [ + {"dimension_values": ["London, England"], "value": 263}, + {"dimension_values": ["Sydney, New South Wales"], "value": 467}, + {"dimension_values": ["Algiers, Algiers Province"], "value": 58}, + {"dimension_values": ["Casablanca, Grand Casablanca"], "value": 71}, + {"dimension_values": ["São Paulo, São Paulo (state)"], "value": 139}, + {"dimension_values": ["Rio de Janeiro, Rio de Janeiro (state)"], "value": 44}, + {"dimension_values": ["Perth, Western Australia"], "value": 180}, + {"dimension_values": ["Berlin, Berlin"], "value": 47}, + {"dimension_values": ["Kolkata, West Bengal"], "value": 85}, + {"dimension_values": ["Phoenix, Arizona"], "value": 39}, + {"dimension_values": ["Lagos, Lagos State"], "value": 40}, + {"dimension_values": ["Dublin, Dublin"], "value": 65}, + {"dimension_values": ["Pune, Maharashtra"], "value": 72}, + {"dimension_values": ["Wollongong, New South Wales"], "value": 43}, + {"dimension_values": ["Christchurch, Canterbury"], "value": 42}, + {"dimension_values": ["Jakarta, Jakarta"], "value": 46}, + {"dimension_values": ["Pretoria, Gauteng"], "value": 54}, + {"dimension_values": ["Buenos Aires, Ciudad Autónoma de Buenos Aires"], "value": 41}, + {"dimension_values": ["Gold Coast, Queensland"], "value": 98}, + {"dimension_values": ["Sunshine Coast, Queensland"], "value": 37}, + {"dimension_values": ["Melbourne, Victoria"], "value": 338}, + {"dimension_values": ["Gurugram, Haryana"], "value": 52}, + {"dimension_values": ["Delhi, Delhi"], "value": 194}, + {"dimension_values": ["Los Angeles, California"], "value": 66}, + {"dimension_values": ["Madrid, Comunidad de Madrid"], "value": 65}, + {"dimension_values": ["Lahore, Punjab"], "value": 41}, + {"dimension_values": ["Brisbane, Queensland"], "value": 160}, + {"dimension_values": ["Adelaide, South Australia"], "value": 93}, + {"dimension_values": ["Canberra, Australian Capital Territory"], "value": 45}, + {"dimension_values": ["Lima, Lima Region"], "value": 43}, + {"dimension_values": ["Istanbul, Istanbul Province"], "value": 57}, + {"dimension_values": ["Toronto, Ontario"], "value": 40}, + {"dimension_values": ["Chennai, Tamil Nadu"], "value": 82}, + {"dimension_values": ["Mexico City, Distrito Federal"], "value": 66}, + {"dimension_values": ["Auckland, Auckland Region"], "value": 98}, + {"dimension_values": ["Cape Town, Western Cape"], "value": 172}, + {"dimension_values": ["New York, New York"], "value": 139}, + {"dimension_values": ["Cairo, Cairo Governorate"], "value": 45}, + {"dimension_values": ["Dubai, Dubai"], "value": 57}, + {"dimension_values": ["Santiago, Santiago Metropolitan Region"], "value": 73}, + {"dimension_values": ["Mumbai, Maharashtra"], "value": 195}, + {"dimension_values": ["Bangalore, Karnataka"], "value": 195}, + {"dimension_values": ["Nairobi, Nairobi"], "value": 50}, + {"dimension_values": ["Johannesburg, Gauteng"], "value": 50}, + {"dimension_values": ["Hyderabad, Telangana"], "value": 49}, + ], } - ] - }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + ] + }, + "id": "17841457631192237/insights/follower_demographics/lifetime", +} expected_breakdown_record_transformed = { "name": "follower_demographics", @@ -379,153 +128,121 @@ "title": "Follower demographics", "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", "value": { - "London, England": 263, - "Sydney, New South Wales": 467, - "Algiers, Algiers Province": 58, - "Casablanca, Grand Casablanca": 71, - "São Paulo, São Paulo (state)": 139, - "Rio de Janeiro, Rio de Janeiro (state)": 44, - "Perth, Western Australia": 180, - "Berlin, Berlin": 47, - "Kolkata, West Bengal": 85, - "Phoenix, Arizona": 39, - "Lagos, Lagos State": 40, - "Dublin, Dublin": 65, - "Pune, Maharashtra": 72, - "Wollongong, New South Wales": 43, - "Christchurch, Canterbury": 42, - "Jakarta, Jakarta": 46, - "Pretoria, Gauteng": 54, - "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, - "Gold Coast, Queensland": 98, - "Sunshine Coast, Queensland": 37, - "Melbourne, Victoria": 338, - "Gurugram, Haryana": 52, - "Delhi, Delhi": 194, - "Los Angeles, California": 66, - "Madrid, Comunidad de Madrid": 65, - "Lahore, Punjab": 41, - "Brisbane, Queensland": 160, - "Adelaide, South Australia": 93, - "Canberra, Australian Capital Territory": 45, - "Lima, Lima Region": 43, - "Istanbul, Istanbul Province": 57, - "Toronto, Ontario": 40, - "Chennai, Tamil Nadu": 82, - "Mexico City, Distrito Federal": 66, - "Auckland, Auckland Region": 98, - "Cape Town, Western Cape": 172, - "New York, New York": 139, - "Cairo, Cairo Governorate": 45, - "Dubai, Dubai": 57, - "Santiago, Santiago Metropolitan Region": 73, - "Mumbai, Maharashtra": 195, - "Bangalore, Karnataka": 195, - "Nairobi, Nairobi": 50, - "Johannesburg, Gauteng": 50, - "Hyderabad, Telangana": 49 + "London, England": 263, + "Sydney, New South Wales": 467, + "Algiers, Algiers Province": 58, + "Casablanca, Grand Casablanca": 71, + "São Paulo, São Paulo (state)": 139, + "Rio de Janeiro, Rio de Janeiro (state)": 44, + "Perth, Western Australia": 180, + "Berlin, Berlin": 47, + "Kolkata, West Bengal": 85, + "Phoenix, Arizona": 39, + "Lagos, Lagos State": 40, + "Dublin, Dublin": 65, + "Pune, Maharashtra": 72, + "Wollongong, New South Wales": 43, + "Christchurch, Canterbury": 42, + "Jakarta, Jakarta": 46, + "Pretoria, Gauteng": 54, + "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, + "Gold Coast, Queensland": 98, + "Sunshine Coast, Queensland": 37, + "Melbourne, Victoria": 338, + "Gurugram, Haryana": 52, + "Delhi, Delhi": 194, + "Los Angeles, California": 66, + "Madrid, Comunidad de Madrid": 65, + "Lahore, Punjab": 41, + "Brisbane, Queensland": 160, + "Adelaide, South Australia": 93, + "Canberra, Australian Capital Territory": 45, + "Lima, Lima Region": 43, + "Istanbul, Istanbul Province": 57, + "Toronto, Ontario": 40, + "Chennai, Tamil Nadu": 82, + "Mexico City, Distrito Federal": 66, + "Auckland, Auckland Region": 98, + "Cape Town, Western Cape": 172, + "New York, New York": 139, + "Cairo, Cairo Governorate": 45, + "Dubai, Dubai": 57, + "Santiago, Santiago Metropolitan Region": 73, + "Mumbai, Maharashtra": 195, + "Bangalore, Karnataka": 195, + "Nairobi, Nairobi": 50, + "Johannesburg, Gauteng": 50, + "Hyderabad, Telangana": 49, }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + "id": "17841457631192237/insights/follower_demographics/lifetime", +} insights_record = { "data": [ - { - "name": "comments", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title1", - "description": "Description1.", - "id": "insta_id/insights/comments/lifetime" - }, - { - "name": "ig_reels_avg_watch_time", - "period": "lifetime", - "values": [ - { - "value": 11900 - } - ], - "title": "2", - "description": "Description2.", - "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime" - }, - { - "name": "ig_reels_video_view_total_time", - "period": "lifetime", - "values": [ - { - "value": 25979677 - } - ], - "title": "title3", - "description": "Description3.", - "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime" - }, - { - "name": "likes", - "period": "lifetime", - "values": [ - { - "value": 102 - } - ], - "title": "title4", - "description": "Description4.", - "id": "insta_id/insights/likes/lifetime" - }, - { - "name": "plays", - "period": "lifetime", - "values": [ - { - "value": 2176 - } - ], - "title": "title5", - "description": "Description5.", - "id": "insta_id/insights/plays/lifetime" - }, - { - "name": "reach", - "period": "lifetime", - "values": [ - { - "value": 1842 - } - ], - "title": "title6", - "description": "Description6.", - "id": "insta_id/insights/reach/lifetime" - }, - { - "name": "saved", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title7", - "description": "Description7.", - "id": "insta_id/insights/saved/lifetime" - }, - { - "name": "shares", - "period": "lifetime", - "values": [ - { - "value": 1 - } - ], - "title": "title8", - "description": "Description8.", - "id": "insta_id/insights/shares/lifetime" - } + { + "name": "comments", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title1", + "description": "Description1.", + "id": "insta_id/insights/comments/lifetime", + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [{"value": 11900}], + "title": "2", + "description": "Description2.", + "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime", + }, + { + "name": "ig_reels_video_view_total_time", + "period": "lifetime", + "values": [{"value": 25979677}], + "title": "title3", + "description": "Description3.", + "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime", + }, + { + "name": "likes", + "period": "lifetime", + "values": [{"value": 102}], + "title": "title4", + "description": "Description4.", + "id": "insta_id/insights/likes/lifetime", + }, + { + "name": "plays", + "period": "lifetime", + "values": [{"value": 2176}], + "title": "title5", + "description": "Description5.", + "id": "insta_id/insights/plays/lifetime", + }, + { + "name": "reach", + "period": "lifetime", + "values": [{"value": 1842}], + "title": "title6", + "description": "Description6.", + "id": "insta_id/insights/reach/lifetime", + }, + { + "name": "saved", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title7", + "description": "Description7.", + "id": "insta_id/insights/saved/lifetime", + }, + { + "name": "shares", + "period": "lifetime", + "values": [{"value": 1}], + "title": "title8", + "description": "Description8.", + "id": "insta_id/insights/shares/lifetime", + }, ] } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 4f371e0fc9cf0..e1bcacfc5dc72 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -5,6 +5,8 @@ import logging +from source_instagram.source import SourceInstagram + from airbyte_cdk.connector_builder.connector_builder_handler import resolve_manifest from airbyte_cdk.models import ( AirbyteStream, @@ -14,7 +16,7 @@ DestinationSyncMode, SyncMode, ) -from source_instagram.source import SourceInstagram + logger = logging.getLogger("airbyte") @@ -24,23 +26,11 @@ account_url_response = { - "data": [ - { - "id": "page_id", - "name": "Airbyte", - "instagram_business_account": { - "id": "instagram_business_account_id" - } - } - ], - "paging": { - "cursors": { - "before": "before", - "after": "after" - } - } + "data": [{"id": "page_id", "name": "Airbyte", "instagram_business_account": {"id": "instagram_business_account_id"}}], + "paging": {"cursors": {"before": "before", "after": "after"}}, } + def test_check_connection_ok(api, requests_mock, some_config): requests_mock.register_uri("GET", account_url, [{"json": account_url_response}]) ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 58fe97e5002fb..74fc743575246 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -6,11 +6,13 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from source_instagram.streams import DatetimeTransformerMixin, InstagramStream, UserInsights from utils import read_full_refresh, read_incremental +from airbyte_cdk.models import SyncMode + + FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -30,7 +32,6 @@ def test_state_is_not_outdated(api, config): assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) - def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" @@ -42,7 +43,6 @@ def test_user_insights_read(api, config, user_insight_data, requests_mock): assert records - @pytest.mark.parametrize( "values,slice_dates,expected", [ diff --git a/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-instatus/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-instatus/main.py b/airbyte-integrations/connectors/source-instatus/main.py index 0e0c5af556b5b..e2352fc98eb12 100644 --- a/airbyte-integrations/connectors/source-instatus/main.py +++ b/airbyte-integrations/connectors/source-instatus/main.py @@ -4,5 +4,6 @@ from source_instatus.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/components.py b/airbyte-integrations/connectors/source-instatus/source_instatus/components.py index 62c6561c19cb3..e589d1f07bb46 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/components.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, List, Mapping, Optional import dpath.util + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig, SubstreamPartitionRouter from airbyte_cdk.sources.declarative.transformations import AddFields diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py index ade50f0a9cddd..ccf55560c448a 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/run.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_instatus import SourceInstatus +from airbyte_cdk.entrypoint import launch + def run(): source = SourceInstatus() diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/source.py b/airbyte-integrations/connectors/source-instatus/source_instatus/source.py index dda77f207308a..2f076f8e1f263 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/source.py +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py index f09c7ff5e83e9..7f6d4b9ceb856 100644 --- a/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-instatus/unit_tests/test_components.py @@ -5,10 +5,11 @@ from unittest.mock import MagicMock import pytest +from source_instatus.components import ListAddFields, UpdatesSubstreamPartitionRouter + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition -from source_instatus.components import ListAddFields, UpdatesSubstreamPartitionRouter @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-intercom/components.py b/airbyte-integrations/connectors/source-intercom/components.py index 4d3670c9490d6..76de447344c9a 100644 --- a/airbyte-integrations/connectors/source-intercom/components.py +++ b/airbyte-integrations/connectors/source-intercom/components.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, Optional, Union import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -18,6 +19,7 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution + RequestInput = Union[str, Mapping[str, str]] @@ -199,7 +201,6 @@ def read_parent_stream( cursor_field: Optional[str], stream_state: Mapping[str, Any], ) -> Iterable[Mapping[str, Any]]: - self.parent_stream.state = stream_state parent_stream_slices_gen = self.parent_stream.stream_slices( diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py index a2a8db7650e54..92dcc44da94e3 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py @@ -6,6 +6,7 @@ import pytest import requests + from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from airbyte_cdk.sources.streams import Stream @@ -75,8 +76,8 @@ def test_sub_slicer(components_module, last_record, expected, records): ], ) def test_rate_limiter(components_module, rate_limit_header, backoff_time): - IntercomRateLimiter = components_module.IntercomRateLimiter + def check_backoff_time(t): """A replacer for original `IntercomRateLimiter.backoff_time`""" assert backoff_time == t, f"Expected {backoff_time}, got {t}" diff --git a/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intruder/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ip2whois/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py index 51f9b12de27dd..e023a7a92b301 100644 --- a/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-iterable/main.py b/airbyte-integrations/connectors/source-iterable/main.py index eef7d894cbc44..5b07bc8070e3b 100644 --- a/airbyte-integrations/connectors/source-iterable/main.py +++ b/airbyte-integrations/connectors/source-iterable/main.py @@ -4,5 +4,6 @@ from source_iterable.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/components.py b/airbyte-integrations/connectors/source-iterable/source_iterable/components.py index 19d218fff055e..71f216c5af376 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/components.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 83fa25aa174b9..f1e78543b423e 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -47,6 +47,7 @@ WebPushSendSkip, ) + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. @@ -54,6 +55,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourceIterable(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index dae02ef017f38..b7e232e5ab569 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -10,18 +10,20 @@ import pendulum import requests +from pendulum.datetime import DateTime +from requests import HTTPError +from requests.exceptions import ChunkedEncodingError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import CheckpointMixin, package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from pendulum.datetime import DateTime -from requests import HTTPError -from requests.exceptions import ChunkedEncodingError from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice from source_iterable.utils import dateutil_parse + EVENT_ROWS_LIMIT = 200 CAMPAIGNS_PER_REQUEST = 20 @@ -202,7 +204,6 @@ def request_params( stream_slice: StreamSlice, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state) params.update( { @@ -250,7 +251,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) return [StreamSlice(start_datetime, self._end_date or pendulum.now("UTC"))] @@ -268,7 +268,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) return RangeSliceGenerator(start_datetime, self._end_date) @@ -303,7 +302,6 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[StreamSlice]]: - start_datetime = self.get_start_date(stream_state) self._adjustable_generator = AdjustableSliceGenerator(start_datetime, self._end_date) return self._adjustable_generator @@ -318,7 +316,6 @@ def read_records( start_time = pendulum.now() for _ in range(self.CHUNKED_ENCODING_ERROR_RETRIES): try: - self.logger.info( f"Processing slice of {(stream_slice.end_date - stream_slice.start_date).total_days()} days for stream {self.name}" ) diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py index 0210920dea23f..f9053d5b44ad7 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py @@ -7,6 +7,7 @@ import pytest import responses + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py index 90bd5b6ab2860..3e54136178d5e 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py @@ -12,11 +12,13 @@ import pendulum import pytest import responses -from airbyte_cdk.models import Type as MessageType from requests.exceptions import ChunkedEncodingError from source_iterable.slice_generators import AdjustableSliceGenerator from source_iterable.source import SourceIterable +from airbyte_cdk.models import Type as MessageType + + TEST_START_DATE = "2020" diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py index 0353d32c22b37..15bdab16414d1 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py @@ -8,9 +8,10 @@ import pendulum import pytest import responses +from source_iterable.source import SourceIterable + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_iterable.source import SourceIterable @pytest.fixture diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py index 661e898d15de8..09a49584e23a8 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_extractors.py @@ -4,9 +4,10 @@ import io import requests -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonlDecoder from source_iterable.components import EventsRecordExtractor +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonlDecoder + def test_events_extraction(): mock_response = requests.Response() diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py index 5c4bcf85a4e93..ff5890eb4030b 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_slice_generator.py @@ -9,6 +9,7 @@ import pytest from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator + TEST_DATE = pendulum.parse("2020-01-01") diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py index e7ed3e0f75ca5..0c44168c77cb3 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py @@ -7,9 +7,10 @@ import pytest import requests import responses +from source_iterable.source import SourceIterable + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice -from source_iterable.source import SourceIterable @responses.activate diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index 1c4d2347ba3d9..4342e91af6b9e 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -6,13 +6,14 @@ import pytest import requests import responses -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice from source_iterable.source import SourceIterable from source_iterable.streams import Campaigns, CampaignsMetrics, Templates from source_iterable.utils import dateutil_parse +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice + def test_campaigns_metrics_csv(): csv_string = "a,b,c,d\n1, 2,,3\n6,,1, 2\n" @@ -190,7 +191,6 @@ def test_path(config, stream, date, slice, expected_path): def test_campaigns_metrics_parse_response(): - stream = CampaignsMetrics(authenticator=None, start_date="2019-10-10T00:00:00") with responses.RequestsMock() as rsps: rsps.add( diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/main.py b/airbyte-integrations/connectors/source-jina-ai-reader/main.py index 33085d47ebf9d..58af77ac1dd2a 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/main.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/main.py @@ -5,5 +5,6 @@ from source_jina_ai_reader.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py index 7604776518b57..ba509deaa9ed1 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/config_migration.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py index bf73db8f9ed61..392ff158b075f 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/source_jina_ai_reader/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py index 2edcb5eb27c76..9825027db046d 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py @@ -5,6 +5,7 @@ from source_jina_ai_reader.config_migration import JinaAiReaderConfigMigration from source_jina_ai_reader.source import SourceJinaAiReader + TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" @@ -16,7 +17,7 @@ def test_should_migrate(): def test__modify_and_save(): source = SourceJinaAiReader() user_config = {"search_prompt": "What is AI"} - expected = {"search_prompt": "What%20is%20AI" } + expected = {"search_prompt": "What%20is%20AI"} modified_config = JinaAiReaderConfigMigration.modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config=user_config) assert modified_config["search_prompt"] == expected["search_prompt"] assert modified_config.get("search_prompt") diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py index e4e2e83377825..fb8a0c348a1db 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/generator.py @@ -7,7 +7,6 @@ from base64 import b64encode from typing import Any, List, Mapping -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from streams import ( DashboardsGenerator, FiltersGenerator, @@ -29,6 +28,8 @@ WorkflowsGenerator, ) +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + class Generator: base_config_path = "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py index 5a295903035b6..365af929548f3 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py +++ b/airbyte-integrations/connectors/source-jira/integration_tests/fixtures/data_generator/streams.py @@ -8,7 +8,6 @@ from typing import Any, Mapping, Optional import requests -from airbyte_cdk.models import SyncMode from source_jira.streams import ( Dashboards, Filters, @@ -31,6 +30,8 @@ WorkflowSchemes, ) +from airbyte_cdk.models import SyncMode + class GeneratorMixin: def get_generate_headers(self): diff --git a/airbyte-integrations/connectors/source-jira/main.py b/airbyte-integrations/connectors/source-jira/main.py index 1885b3974defb..528d513d74889 100644 --- a/airbyte-integrations/connectors/source-jira/main.py +++ b/airbyte-integrations/connectors/source-jira/main.py @@ -4,5 +4,6 @@ from source_jira.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py b/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py index 2a30cc6665410..ff56146ada56f 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/components/extractors.py @@ -3,9 +3,10 @@ from dataclasses import dataclass from typing import Any, List, Mapping -from airbyte_cdk.sources.declarative.extractors import DpathExtractor from requests_cache import Response +from airbyte_cdk.sources.declarative.extractors import DpathExtractor + @dataclass class LabelsRecordExtractor(DpathExtractor): diff --git a/airbyte-integrations/connectors/source-jira/source_jira/run.py b/airbyte-integrations/connectors/source-jira/source_jira/run.py index 537430466f7ae..8f5465ccd6c74 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/run.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/run.py @@ -7,10 +7,11 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_jira import SourceJira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index c72c2d5664fab..1e39ab1230804 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -5,6 +5,9 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from pydantic import ValidationError +from requests.exceptions import InvalidURL + from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource @@ -12,8 +15,6 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from pydantic import ValidationError -from requests.exceptions import InvalidURL from .streams import IssueFields, Issues, PullRequests from .utils import read_full_refresh diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 1d9f622bd4a71..b9ef14e0cce3c 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -12,6 +12,8 @@ import pendulum import requests +from requests.exceptions import HTTPError + from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import CheckpointMixin, Stream from airbyte_cdk.sources.streams.checkpoint.checkpoint_reader import FULL_REFRESH_COMPLETE_STATE @@ -21,11 +23,11 @@ from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from requests.exceptions import HTTPError from source_jira.type_transfromer import DateTimeTransformer from .utils import read_full_refresh, read_incremental, safe_max + API_VERSION = 3 @@ -173,7 +175,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class FullRefreshJiraStream(JiraStream): - """ This is a temporary solution to avoid incorrect state handling. See comments below for more info: diff --git a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py index 16fdfb000a4c6..efb3bca61278c 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.utils.transform import TypeTransformer + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-jira/source_jira/utils.py b/airbyte-integrations/connectors/source-jira/source_jira/utils.py index f85c7363ff979..af6adbcf91087 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/utils.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/utils.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.streams import Stream + _NO_STATE: MutableMapping[str, Any] = {} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py index 215560d88cf17..b25b3acd3ba28 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py @@ -11,6 +11,7 @@ from responses import matchers from source_jira.source import SourceJira + ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py index c0306ea87a4a1..2d8f2b2127414 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py @@ -6,6 +6,8 @@ from unittest import TestCase import freezegun +from source_jira import SourceJira + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -20,7 +22,7 @@ ) from airbyte_cdk.test.state_builder import StateBuilder from integration.config import ConfigBuilder -from source_jira import SourceJira + _STREAM_NAME = "issues" _API_TOKEN = "api_token" @@ -40,6 +42,7 @@ def _response_template() -> Dict[str, Any]: with open(os.path.join(os.path.dirname(__file__), "..", "responses", "issues.json")) as response_file_handler: return json.load(response_file_handler) + def _create_response() -> HttpResponseBuilder: return create_response_builder( response_template=_response_template(), @@ -60,15 +63,19 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: config = _create_config().build() datetime_with_timezone = "2023-11-01T00:00:00.000-0800" timestamp_with_timezone = 1698825600000 - state = StateBuilder().with_stream_state( - "issues", - { - "use_global_cursor":False, - "state": {"updated": datetime_with_timezone}, - "lookback_window": 2, - "states": [{"partition":{"parent_slice":{},"project_id":"10025"},"cursor":{"updated": datetime_with_timezone}}] - } - ).build() + state = ( + StateBuilder() + .with_stream_state( + "issues", + { + "use_global_cursor": False, + "state": {"updated": datetime_with_timezone}, + "lookback_window": 2, + "states": [{"partition": {"parent_slice": {}, "project_id": "10025"}, "cursor": {"updated": datetime_with_timezone}}], + }, + ) + .build() + ) http_mocker.get( HttpRequest( f"https://{_DOMAIN}/rest/api/3/search", @@ -77,7 +84,7 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: "jql": f"updated >= {timestamp_with_timezone} ORDER BY updated asc", "expand": "renderedFields,transitions,changelog", "maxResults": "50", - } + }, ), _create_response().with_record(_create_record()).with_record(_create_record()).build(), ) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py index e24edba48d2fe..50e20916d0930 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_components.py @@ -4,10 +4,11 @@ import pytest import requests -from airbyte_cdk.sources.declarative.types import StreamSlice from source_jira.components.extractors import LabelsRecordExtractor from source_jira.components.partition_routers import SprintIssuesSubstreamPartitionRouter +from airbyte_cdk.sources.declarative.types import StreamSlice + @pytest.mark.parametrize( "json_response, expected_output", diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py index be35d1a7f0d5c..e072cc6616ed3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py @@ -6,9 +6,10 @@ import pytest import responses -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_jira.source import SourceJira +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @responses.activate def test_streams(config): diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index b6d05c6f4fbc0..fd4f10dfd98ac 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -5,16 +5,17 @@ import pendulum import pytest import responses -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import read -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from conftest import find_stream from responses import matchers from source_jira.source import SourceJira from source_jira.streams import IssueFields, Issues, PullRequests from source_jira.utils import read_full_refresh, read_incremental +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + @responses.activate def test_application_roles_stream_401_error(config, caplog): @@ -50,9 +51,7 @@ def test_application_roles_stream_http_error(config, application_roles_response) responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/applicationrole", json={"error": "not found"}, status=404) stream = find_stream("application_roles", config) - with pytest.raises( - AirbyteTracedException, match="Not found. The requested resource was not found on the server" - ): + with pytest.raises(AirbyteTracedException, match="Not found. The requested resource was not found on the server"): list(read_full_refresh(stream)) @@ -82,10 +81,7 @@ def test_board_stream_forbidden(config, boards_response, caplog): ) stream = find_stream("boards", config) - with pytest.raises( - AirbyteTracedException, - match="Forbidden. You don't have permission to access this resource." - ): + with pytest.raises(AirbyteTracedException, match="Forbidden. You don't have permission to access this resource."): list(read_full_refresh(stream)) @@ -96,7 +92,7 @@ def test_dashboards_stream(config, dashboards_response): f"https://{config['domain']}/rest/api/3/dashboard", json=dashboards_response, ) - + stream = find_stream("dashboards", config) records = list(read_full_refresh(stream)) @@ -132,7 +128,7 @@ def test_groups_stream(config, groups_response): def test_issues_fields_stream(config, mock_fields_response): stream = find_stream("issue_fields", config) records = list(read_full_refresh(stream)) - + assert len(records) == 6 assert len(responses.calls) == 1 @@ -149,7 +145,7 @@ def test_python_issues_fields_ids_by_name(config, mock_fields_response): "Issue Type": ["issuetype"], "Parent": ["parent"], "Issue Type2": ["issuetype2"], - "Issue Type3": ["issuetype3"] + "Issue Type3": ["issuetype3"], } assert expected_ids_by_name == stream.field_ids_by_name() @@ -400,7 +396,9 @@ def test_screen_tabs_stream(config, mock_screen_response, screen_tabs_response): @responses.activate def test_sprints_stream(config, mock_board_response, mock_sprints_response): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build() + ) assert len(output.records) == 3 assert len(responses.calls) == 4 @@ -417,7 +415,7 @@ def test_board_does_not_support_sprints(config, mock_board_response, sprints_res responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50", json={"errorMessages": ["The board does not support sprints"], "errors": {}}, - status=400 + status=400, ) responses.add( responses.GET, @@ -443,7 +441,11 @@ def test_sprint_issues_stream(config, mock_board_response, mock_fields_response, json=sprints_issues_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build(), + ) assert len(output.records) == 3 assert len(responses.calls) == 8 @@ -585,7 +587,7 @@ def test_avatars_stream_should_retry(config, caplog): responses.GET, f"https://{config['domain']}/rest/api/3/avatar/{slice}/system", json={"errorMessages": ["The error message"], "errors": {}}, - status=400 + status=400, ) stream = find_stream("avatars", config) @@ -622,9 +624,11 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project assert "non_empty_field" in records[0]["fields"] assert len(responses.calls) == 3 - error_message = ("Stream `issues`. An error occurred, details: The user doesn't have " - 'permission to the project. Please grant the user to the project. Errors: ' - '["The value \'3\' does not exist for the field \'project\'."]') + error_message = ( + "Stream `issues`. An error occurred, details: The user doesn't have " + "permission to the project. Please grant the user to the project. Errors: " + "[\"The value '3' does not exist for the field 'project'.\"]" + ) assert error_message in caplog.messages @@ -632,23 +636,24 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project @pytest.mark.parametrize( "status_code, response_errorMessages, expected_log_message", ( - (400, - ["The value 'incorrect_project' does not exist for the field 'project'."], - ( - "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." - " Please grant the user to the project. " - "Errors: [\"The value \'incorrect_project\' does not exist for the field \'project\'.\"]" - ) - ), + ( + 400, + ["The value 'incorrect_project' does not exist for the field 'project'."], ( - 403, - ["The value 'incorrect_project' doesn't have permission for the field 'project'."], - ( - 'Stream `issues`. An error occurred, details:' - ' Errors: ["The value \'incorrect_project\' doesn\'t have permission for the field \'project\'."]' - ) + "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." + " Please grant the user to the project. " + "Errors: [\"The value 'incorrect_project' does not exist for the field 'project'.\"]" ), - ) + ), + ( + 403, + ["The value 'incorrect_project' doesn't have permission for the field 'project'."], + ( + "Stream `issues`. An error occurred, details:" + " Errors: [\"The value 'incorrect_project' doesn't have permission for the field 'project'.\"]" + ), + ), + ), ) def test_python_issues_stream_skip_on_http_codes_error_handling(config, status_code, response_errorMessages, expected_log_message, caplog): responses.add( @@ -689,8 +694,7 @@ def test_python_issues_stream_updated_state(config): stream = Issues(**args) updated_state = stream._get_updated_state( - current_stream_state={"updated": "2021-01-01T00:00:00Z"}, - latest_record={"updated": "2021-01-02T00:00:00Z"} + current_stream_state={"updated": "2021-01-01T00:00:00Z"}, latest_record={"updated": "2021-01-02T00:00:00Z"} ) assert updated_state == {"updated": "2021-01-02T00:00:00Z"} @@ -703,7 +707,7 @@ def test_python_issues_stream_updated_state(config): ("pullrequest={dataType=pullrequest, state=thestate, stateCount=1}", True), ("pullrequest={dataType=pullrequest, state=thestate, stateCount=0}", False), ("{}", False), - ) + ), ) def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pull_request): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) @@ -721,7 +725,9 @@ def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pul @responses.activate -def test_python_pull_requests_stream_has_pull_request(config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter): +def test_python_pull_requests_stream_has_pull_request( + config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter +): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} issues_stream = Issues(**args) @@ -822,7 +828,11 @@ def test_project_permissions_stream(config, mock_non_deleted_projects_responses, @responses.activate def test_project_email_stream(config, mock_non_deleted_projects_responses, mock_project_emails): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -836,7 +846,11 @@ def test_project_components_stream(config, mock_non_deleted_projects_responses, json=project_components_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -850,7 +864,11 @@ def test_permissions_stream(config, permissions_response): json=permissions_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build(), + ) assert len(output.records) == 1 assert len(responses.calls) == 1 @@ -869,7 +887,9 @@ def test_labels_stream(config, labels_response): json={}, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build() + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -955,38 +975,33 @@ def test_project_versions_stream(config, mock_non_deleted_projects_responses, pr @pytest.mark.parametrize( "stream, expected_records_number, expected_calls_number, log_message", [ - ( - "issues", - 2, - 4, - "The user doesn't have permission to the project. Please grant the user to the project." - ), + ("issues", 2, 4, "The user doesn't have permission to the project. Please grant the user to the project."), ( "issue_custom_field_contexts", 2, 4, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_contexts`. An error occurred, details: ['Not found issue custom field context for issue fields issuetype2']. Skipping for now. ", ), ( "issue_custom_field_options", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_options`. An error occurred, details: ['Not found issue custom field options for issue fields issuetype3']. Skipping for now. ", ), ( "issue_watchers", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_watchers`. An error occurred, details: ['Not found watchers for issue TESTKEY13-2']. Skipping for now. ", ), ( "project_email", 4, 4, - "Forbidden. You don't have permission to access this resource." + "Forbidden. You don't have permission to access this resource.", # "Stream `project_email`. An error occurred, details: ['No access to emails for project 3']. Skipping for now. ", ), ], @@ -1009,7 +1024,9 @@ def test_skip_slice( log_message, ): config["projects"] = config.get("projects", []) + ["Project3", "Project4"] - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build() + ) assert len(output.records) == expected_records_number assert len(responses.calls) == expected_calls_number diff --git a/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-k6-cloud/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kafka/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-klaviyo/main.py b/airbyte-integrations/connectors/source-klaviyo/main.py index 5b8c871c3d7d1..bda84b0215c4b 100644 --- a/airbyte-integrations/connectors/source-klaviyo/main.py +++ b/airbyte-integrations/connectors/source-klaviyo/main.py @@ -4,5 +4,6 @@ from source_klaviyo.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py index f7938ab9e6e75..c25c6a8c4983a 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py @@ -5,11 +5,12 @@ import logging from typing import Dict, Optional +from requests import HTTPError, codes + from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING -from requests import HTTPError, codes class KlaviyoAvailabilityStrategy(HttpAvailabilityStrategy): diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py index c4088bf6c7b34..bd412653d7cf7 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/included_fields_extractor.py @@ -7,6 +7,7 @@ import dpath import requests + from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py index 270f097043497..411cb965fc935 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/components/klaviyo_error_handler.py @@ -3,10 +3,11 @@ from typing import Optional, Union import requests +from requests.exceptions import InvalidURL + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, ResponseAction -from requests.exceptions import InvalidURL class KlaviyoErrorHandler(DefaultErrorHandler): diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py index 4607ff3dea197..54eda7f3b7cd6 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/run.py @@ -8,9 +8,10 @@ import traceback from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_klaviyo import SourceKlaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 1cff0bffa204e..487d34bc43079 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -8,6 +8,8 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum +from requests import Response + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import WaitTimeFromHeaderBackoffStrategy from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy @@ -16,7 +18,6 @@ from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler, HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction -from requests import Response from .availability_strategy import KlaviyoAvailabilityStrategy from .exceptions import KlaviyoBackoffError @@ -128,7 +129,6 @@ def read_records( stream_slice: Optional[Mapping[str, Any]] = None, stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: - current_state = self.state or {} try: for record in super().read_records(sync_mode, cursor_field, stream_slice, current_state): @@ -263,7 +263,6 @@ def _set_campaign_message(self, record: Mapping[str, Any]) -> None: record["campaign_message"] = campaign_message_response.json().get("data") def get_error_handler(self) -> ErrorHandler: - error_mapping = DEFAULT_ERROR_MAPPING | { 404: ErrorResolution(ResponseAction.IGNORE, FailureType.config_error, "Resource not found. Ignoring.") } diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py index 8789782701546..a217637afe7d8 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py @@ -5,7 +5,7 @@ class KlaviyoConfigBuilder: def __init__(self) -> None: - self._config = {"api_key":"an_api_key","start_date":"2021-01-01T00:00:00Z"} + self._config = {"api_key": "an_api_key", "start_date": "2021-01-01T00:00:00Z"} def with_start_date(self, start_date: datetime) -> "KlaviyoConfigBuilder": self._config["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py index 969306e0d75c2..2395a79040fa4 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py @@ -3,6 +3,8 @@ from typing import Any, Dict, Optional from unittest import TestCase +from source_klaviyo import SourceKlaviyo + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -17,7 +19,7 @@ find_template, ) from integration.config import KlaviyoConfigBuilder -from source_klaviyo import SourceKlaviyo + _ENDPOINT_TEMPLATE_NAME = "profiles" _START_DATE = datetime.datetime(2021, 1, 1, tzinfo=datetime.timezone.utc) @@ -37,11 +39,11 @@ def _a_profile_request(start_date: datetime) -> HttpRequest: return HttpRequest( url=f"https://a.klaviyo.com/api/profiles", query_params={ - "additional-fields[profile]": "predictive_analytics", - "page[size]": "100", - "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", - "sort": "updated" - } + "additional-fields[profile]": "predictive_analytics", + "page[size]": "100", + "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", + "sort": "updated", + }, ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py index 211a158290e35..df84e66f80295 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py @@ -34,26 +34,26 @@ def extractor(mock_config, mock_field_path, mock_decoder): return KlaviyoIncludedFieldExtractor(mock_field_path, mock_config, mock_decoder) -@patch('dpath.get') -@patch('dpath.values') +@patch("dpath.get") +@patch("dpath.values") def test_extract_records_by_path(mock_values, mock_get, extractor, mock_response, mock_decoder): - mock_values.return_value = [{'key': 'value'}] - mock_get.return_value = {'key': 'value'} - mock_decoder.decode.return_value = {'data': 'value'} + mock_values.return_value = [{"key": "value"}] + mock_get.return_value = {"key": "value"} + mock_decoder.decode.return_value = {"data": "value"} - field_paths = ['data'] + field_paths = ["data"] records = list(extractor.extract_records_by_path(mock_response, field_paths)) - assert records == [{'key': 'value'}] + assert records == [{"key": "value"}] mock_values.return_value = [] mock_get.return_value = None - records = list(extractor.extract_records_by_path(mock_response, ['included'])) + records = list(extractor.extract_records_by_path(mock_response, ["included"])) assert records == [] def test_update_target_records_with_included(extractor): - target_records = [{'relationships': {'type1': {'data': {'id': 1}}}}] - included_records = [{'id': 1, 'type': 'type1', 'attributes': {'key': 'value'}}] + target_records = [{"relationships": {"type1": {"data": {"id": 1}}}}] + included_records = [{"id": 1, "type": "type1", "attributes": {"key": "value"}}] updated_records = list(extractor.update_target_records_with_included(target_records, included_records)) - assert updated_records[0]['relationships']['type1']['data'] == {'id': 1, 'key': 'value'} + assert updated_records[0]["relationships"]["type1"]["data"] == {"id": 1, "key": "value"} diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py index 052a93ae0f30d..7cf3138d22f90 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py @@ -4,6 +4,8 @@ from unittest.mock import MagicMock +from source_klaviyo.components.per_partition_state_migration import PerPartitionToSingleStateMigration + from airbyte_cdk.sources.declarative.models import ( CustomRetriever, DatetimeBasedCursor, @@ -14,7 +16,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory -from source_klaviyo.components.per_partition_state_migration import PerPartitionToSingleStateMigration + factory = ModelToComponentFactory() @@ -26,14 +28,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): input_state = { "states": [ - { - "partition": {"parent_id": "13506132"}, - "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"} - }, - { - "partition": {"parent_id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, + {"partition": {"parent_id": "13506132"}, "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"}}, + {"partition": {"parent_id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] } @@ -61,14 +57,10 @@ def _migrator(): parent_key="{{ parameters['parent_key_id'] }}", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ) - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py index a25db137a57c8..17cc35eb63524 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py @@ -7,11 +7,13 @@ import pendulum import pytest -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from integration.config import KlaviyoConfigBuilder from source_klaviyo.source import SourceKlaviyo +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + logger = logging.getLogger("airbyte") @@ -29,16 +31,12 @@ def _source() -> SourceKlaviyo: ( 400, False, - ( - "Bad request. Please check your request parameters." - ), + ("Bad request. Please check your request parameters."), ), ( 403, False, - ( - "Please provide a valid API key and make sure it has permissions to read specified streams." - ), + ("Please provide a valid API key and make sure it has permissions to read specified streams."), ), ), ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py index 2ed50d5e46f41..f501f5aeaa68f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -14,11 +14,6 @@ import pendulum import pytest import requests -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from dateutil.relativedelta import relativedelta from integration.config import KlaviyoConfigBuilder from pydantic import BaseModel @@ -26,6 +21,13 @@ from source_klaviyo.source import SourceKlaviyo from source_klaviyo.streams import Campaigns, CampaignsDetailed, IncrementalKlaviyoStream, KlaviyoStream +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + _ANY_ATTEMPT_COUNT = 123 API_KEY = "some_key" START_DATE = pendulum.datetime(2020, 10, 10) @@ -36,6 +38,7 @@ EVENTS_STREAM_STATE_DATE = (datetime.fromisoformat(EVENTS_STREAM_CONFIG_START_DATE) + relativedelta(years=1)).isoformat() EVENTS_STREAM_TESTING_FREEZE_TIME = "2023-12-12 12:00:00" + def get_step_diff(provided_date: str) -> int: """ This function returns the difference in weeks between provided date and freeze time. @@ -44,6 +47,7 @@ def get_step_diff(provided_date: str) -> int: freeze_date = datetime.strptime(EVENTS_STREAM_TESTING_FREEZE_TIME, "%Y-%m-%d %H:%M:%S") return (freeze_date - provided_date).days // 7 + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceKlaviyo(CatalogBuilder().build(), KlaviyoConfigBuilder().build(), StateBuilder().build()) matches_by_name = [stream_config for stream_config in source.streams(config) if stream_config.name == stream_name] @@ -84,9 +88,7 @@ def path(self, **kwargs) -> str: class TestKlaviyoStream: def test_request_headers(self): stream = SomeStream(api_key=API_KEY) - expected_headers = { - "Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}" - } + expected_headers = {"Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}"} assert stream.request_headers() == expected_headers @pytest.mark.parametrize( @@ -148,9 +150,7 @@ def test_availability_strategy(self): "This is most likely due to insufficient permissions on the credentials in use. " "Try to create and use an API key with read permission for the 'some_stream' stream granted" ) - reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes( - stream, None, None, None - ) + reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes(stream, None, None, None) assert expected_status_code in reasons_for_unavailable_status_codes assert reasons_for_unavailable_status_codes[expected_status_code] == expected_message @@ -173,14 +173,11 @@ def test_backoff_time_large_retry_after(self): response_mock.headers = {"Retry-After": retry_after} with pytest.raises(AirbyteTracedException) as e: stream.get_backoff_strategy().backoff_time(response_mock, _ANY_ATTEMPT_COUNT) - error_message = ( - "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." - ) + error_message = "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." assert str(e.value) == error_message class TestIncrementalKlaviyoStream: - @staticmethod def generate_api_urls(start_date_str: str) -> list[(str, str)]: """ @@ -197,12 +194,12 @@ def generate_api_urls(start_date_str: str) -> list[(str, str)]: end_date = current_date start_date_str = start_date.strftime("%Y-%m-%dT%H:%M:%S") + start_date.strftime("%z") end_date_str = end_date.strftime("%Y-%m-%dT%H:%M:%S") + end_date.strftime("%z") - base_url = 'https://a.klaviyo.com/api/events' + base_url = "https://a.klaviyo.com/api/events" query_params = { - 'fields[metric]': 'name,created,updated,integration', - 'include': 'metric', - 'filter': f'greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})', - 'sort': 'datetime' + "fields[metric]": "name,created,updated,integration", + "include": "metric", + "filter": f"greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})", + "sort": "datetime", } encoded_query = urllib.parse.urlencode(query_params) encoded_url = f"{base_url}?{encoded_query}" @@ -289,7 +286,6 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso latest_record={stream.cursor_field: latest_cursor}, ) == {stream.cursor_field: expected_cursor} - @freezegun.freeze_time("2023-12-12 12:00:00") @pytest.mark.parametrize( # expected_amount_of_results: we put 1 record for every request @@ -297,20 +293,20 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso ( ( # we pick the state - EVENTS_STREAM_CONFIG_START_DATE, - EVENTS_STREAM_STATE_DATE, - get_step_diff(EVENTS_STREAM_STATE_DATE) + 1 # adding last request + EVENTS_STREAM_CONFIG_START_DATE, + EVENTS_STREAM_STATE_DATE, + get_step_diff(EVENTS_STREAM_STATE_DATE) + 1, # adding last request ), ( - # we pick the config start date - EVENTS_STREAM_CONFIG_START_DATE, - None, - get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1 # adding last request + # we pick the config start date + EVENTS_STREAM_CONFIG_START_DATE, + None, + get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1, # adding last request ), ( - "", - "", - get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1 # adding last request + "", + "", + get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1, # adding last request ), ), ) @@ -370,16 +366,13 @@ class TestSemiIncrementalKlaviyoStream: ) def test_read_records(self, start_date, stream_state, input_records, expected_records, requests_mock): stream = get_stream_by_name("metrics", CONFIG | {"start_date": start_date}) - requests_mock.register_uri( - "GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records} - ) + requests_mock.register_uri("GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records}) stream.stream_state = {stream.cursor_field: stream_state if stream_state else start_date} records = get_records(stream=stream, sync_mode=SyncMode.incremental) assert records == expected_records class TestProfilesStream: - def test_read_records(self, requests_mock): stream = get_stream_by_name("profiles", CONFIG) json = { @@ -617,9 +610,9 @@ def test_stream_slices(self): ) def test_request_params(self, stream_state, stream_slice, next_page_token, expected_params): stream = Campaigns(api_key=API_KEY) - assert stream.request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ) == expected_params + assert ( + stream.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) == expected_params + ) class TestCampaignsDetailedStream: @@ -647,7 +640,9 @@ def test_set_recipient_count_not_found(self, requests_mock): mocked_response.ok = False mocked_response.status_code = 404 mocked_response.json.return_value = {} - with patch.object(stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response)): + with patch.object( + stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response) + ): stream._set_recipient_count(record) assert record["estimated_recipient_count"] == 0 diff --git a/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kyriba/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kyriba/main.py b/airbyte-integrations/connectors/source-kyriba/main.py index cd0b8f1f2f3e7..33236c5afe078 100644 --- a/airbyte-integrations/connectors/source-kyriba/main.py +++ b/airbyte-integrations/connectors/source-kyriba/main.py @@ -4,5 +4,6 @@ from source_kyriba.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py index cac3eb31f5cfa..81155b78ea894 100644 --- a/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py +++ b/airbyte-integrations/connectors/source-kyriba/source_kyriba/source.py @@ -8,6 +8,7 @@ import backoff import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py index 58ff037abc679..9e53da81f4d23 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py @@ -20,10 +20,7 @@ def patch_base_class(mocker): def test_stream_slices(patch_base_class): stream = BankBalancesStream(**config()) - account_uuids = [ - {"account_uuid": "first"}, - {"account_uuid": "second"} - ] + account_uuids = [{"account_uuid": "first"}, {"account_uuid": "second"}] stream.get_account_uuids = MagicMock(return_value=account_uuids) stream.start_date = date(2022, 1, 1) stream.end_date = date(2022, 1, 2) @@ -43,7 +40,7 @@ def test_stream_slices(patch_base_class): { "account_uuid": "second", "date": "2022-01-02", - } + }, ] slices = stream.stream_slices() assert slices == expected diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py index ae0b83acd3a4b..904b7e0564ba4 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_cash_flows.py @@ -7,10 +7,11 @@ from unittest.mock import MagicMock import requests -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyriba.source import CashFlows +from airbyte_cdk.models import SyncMode + from .test_streams import config diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py index 5ddcaa10b8d98..cd6223667e061 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyriba.source import IncrementalKyribaStream +from airbyte_cdk.models import SyncMode + from .test_streams import config diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py index 1bda3981cbd62..6be7e9b33a63a 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py @@ -6,6 +6,7 @@ from source_kyriba.source import KyribaClient, SourceKyriba + config = { "username": "username", "password": "password", @@ -13,6 +14,7 @@ "start_date": "2022-01-01", } + def test_check_connection(mocker): source = SourceKyriba() KyribaClient.login = MagicMock() diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py index 85f363267d8bd..99d6da8a1a895 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_streams.py @@ -7,9 +7,10 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_kyriba.source import KyribaClient, KyribaStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.fixture def patch_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py index 77f078a20c155..6b67245046785 100644 --- a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-kyve/main.py b/airbyte-integrations/connectors/source-kyve/main.py index a3740b34d958b..cd5ae9e0c5f34 100644 --- a/airbyte-integrations/connectors/source-kyve/main.py +++ b/airbyte-integrations/connectors/source-kyve/main.py @@ -4,5 +4,6 @@ from source_kyve.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py index 2ec43d5c80dbe..d9f7ba7806d71 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -39,7 +40,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: pools = config.get("pool_ids").split(",") start_ids = config.get("start_ids").split(",") - for (pool_id, start_id) in zip(pools, start_ids): + for pool_id, start_id in zip(pools, start_ids): response = requests.get(f"{config['url_base']}/kyve/query/v1beta1/pool/{pool_id}") pool_data = response.json().get("pool").get("data") diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py index 6688a5832f2f2..d2096615d53a9 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py @@ -8,10 +8,12 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream from source_kyve.utils import query_endpoint + logger = logging.getLogger("airbyte") # 1: Arweave diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py index ea824656df583..1a87d748b08f7 100644 --- a/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_kyve.source import KYVEStream as IncrementalKyveStream +from airbyte_cdk.models import SyncMode + from . import config, pool_data diff --git a/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-launchdarkly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/main.py b/airbyte-integrations/connectors/source-linkedin-ads/main.py index 899a7e8614a4e..bf3f38b6d9fc2 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/main.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/main.py @@ -4,5 +4,6 @@ from source_linkedin_ads.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py index 2fcb05b7f7fb7..1ecc55bcccbaf 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/components.py @@ -12,6 +12,8 @@ import pendulum import requests +from isodate import Duration, parse_duration + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import CursorFactory, DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString @@ -31,7 +33,6 @@ from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS -from isodate import Duration, parse_duration from .utils import ANALYTICS_FIELDS_V2, FIELDS_CHUNK_SIZE, transform_data diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py index 276afed6da58c..5bfbe9788132f 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/config_migrations.py @@ -10,6 +10,7 @@ from airbyte_cdk import AirbyteEntrypoint, Source, create_connector_config_control_message from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 6813aaf34da14..3933f7dca212a 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -14,6 +14,7 @@ from .utils import update_specific_key + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index b10d822ae9ca2..5237fc476f4bf 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -7,6 +7,7 @@ import pendulum as pdm + # replace `pivot` with `_pivot`, to allow redshift normalization, # since `pivot` is a reserved keyword for Destination Redshift, # on behalf of https://github.com/airbytehq/airbyte/issues/13018, diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index d935759843862..69255efc6a072 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -8,6 +8,7 @@ from source_linkedin_ads.source import SourceLinkedinAds + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py index 2fa7fbf0dd34f..3ac0fae777714 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py @@ -18,6 +18,7 @@ StreamSlice, ) + logger = logging.getLogger("airbyte") @@ -35,7 +36,13 @@ def mock_response(): @pytest.fixture def mock_analytics_cursor_params(): - return {"start_datetime": MagicMock(), "cursor_field": MagicMock(), "datetime_format": "%s", "config": MagicMock(), "parameters": MagicMock()} + return { + "start_datetime": MagicMock(), + "cursor_field": MagicMock(), + "datetime_format": "%s", + "config": MagicMock(), + "parameters": MagicMock(), + } @pytest.fixture diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py index 931da4bfbbcdf..17ee2e8dde596 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -6,13 +6,15 @@ import pytest import requests +from conftest import find_stream +from source_linkedin_ads.source import SourceLinkedinAds + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator -from conftest import find_stream -from source_linkedin_ads.source import SourceLinkedinAds + logger = logging.getLogger("airbyte") @@ -90,7 +92,9 @@ def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> No @pytest.mark.parametrize("error_code", [429, 500, 503]) def test_should_retry_on_error(self, error_code, requests_mock, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) stream = find_stream("accounts", TEST_CONFIG) requests_mock.register_uri( @@ -102,11 +106,10 @@ def test_should_retry_on_error(self, error_code, requests_mock, mocker): def test_custom_streams(self): config = {"ad_analytics_reports": [{"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}], **TEST_CONFIG} - ad_campaign_analytics = find_stream('ad_campaign_analytics', config) + ad_campaign_analytics = find_stream("ad_campaign_analytics", config) for stream in self._instance._create_custom_ad_analytics_streams(config=config): assert isinstance(stream, type(ad_campaign_analytics)) - @pytest.mark.parametrize( "stream_name, expected", [ @@ -136,26 +139,23 @@ def test_path(self, stream_name, expected): @pytest.mark.parametrize( ("status_code", "is_connection_successful", "error_msg"), ( - ( - 400, - False, - ( - "Bad request. Please check your request parameters." - ), - ), - ( - 403, - False, - ( - "Forbidden. You don't have permission to access this resource." - ), - ), - (200, True, None), + ( + 400, + False, + ("Bad request. Please check your request parameters."), + ), + ( + 403, + False, + ("Forbidden. You don't have permission to access this resource."), + ), + (200, True, None), ), ) def test_check_connection(self, requests_mock, status_code, is_connection_successful, error_msg, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", - side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) json = {"elements": [{"data": []}] * 500} if 200 >= status_code < 300 else {} requests_mock.register_uri( diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py index 197f43ea8e648..01c828362c8f0 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py @@ -6,11 +6,13 @@ import os from typing import Any, Mapping -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from conftest import find_stream from freezegun import freeze_time +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + + # Test input arguments for the `make_analytics_slices` TEST_KEY_VALUE_MAP = {"camp_id": "id"} TEST_START_DATE = "2021-08-01" @@ -50,7 +52,8 @@ def test_read_records(requests_mock): requests_mock.get("https://api.linkedin.com/rest/adAccounts", json={"elements": [{"id": 1}]}) requests_mock.get( "https://api.linkedin.com/rest/adAccounts/1/adCampaigns?q=search&search=(status:(values:List(ACTIVE,PAUSED,ARCHIVED,COMPLETED,CANCELED,DRAFT,PENDING_DELETION,REMOVED)))", - json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}) + json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}, + ) requests_mock.get( "https://api.linkedin.com/rest/adAnalytics", [ diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py index e116afe2bbbdd..83ca7aaf25473 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -2,7 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -""" This is the example of input record for the test_tranform_data. """ +"""This is the example of input record for the test_tranform_data.""" + input_test_data = [ { "targetingCriteria": { @@ -65,7 +66,7 @@ } }, "pivot": "TEST_PIVOT_VALUE", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] @@ -144,6 +145,6 @@ "end_date": "2021-08-13", "_pivot": "TEST_PIVOT_VALUE", "string_of_pivot_values": "TEST_PIVOT_VALUE_1,TEST_PIVOT_VALUE_2", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py index 73be5f6a05645..c62676cfc0ff3 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py @@ -74,8 +74,8 @@ "nested_dictionary_update", "list_of_dictionaries_update", "excluded_key_in_nested_dict", - "nested_list_with_mixed_types" - ] + "nested_list_with_mixed_types", + ], ) def test_update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys, expected_output): result = update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys) diff --git a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-linnworks/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-linnworks/main.py b/airbyte-integrations/connectors/source-linnworks/main.py index ee964c061ce04..e6b07506ae4fb 100644 --- a/airbyte-integrations/connectors/source-linnworks/main.py +++ b/airbyte-integrations/connectors/source-linnworks/main.py @@ -4,5 +4,6 @@ from source_linnworks.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 7192e2ec58267..32761a344de52 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -7,6 +7,7 @@ import pendulum import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator @@ -48,7 +49,6 @@ def get_auth_header(self) -> Mapping[str, Any]: return {"Authorization": self.get_access_token()} def get_access_token(self): - if self.token_has_expired(): t0 = pendulum.now() token, expires_in, server = self.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py index a3935d7198443..fe7733524ec35 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py @@ -11,11 +11,12 @@ import pendulum import requests import vcr -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from requests.auth import AuthBase from vcr.cassette import Cassette +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator + class LinnworksStream(HttpStream, ABC): http_method = "POST" diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py index d0c1081e0625f..1ab8760bc5051 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py @@ -11,9 +11,10 @@ import pytest import requests import vcr +from source_linnworks.streams import IncrementalLinnworksStream, ProcessedOrderDetails, ProcessedOrders + from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.streams.http.http import HttpSubStream -from source_linnworks.streams import IncrementalLinnworksStream, ProcessedOrderDetails, ProcessedOrders @pytest.fixture diff --git a/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lokalise/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py index 88fa5a9d4eaa7..70e25009b6b48 100644 --- a/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-looker/integration_tests/acceptance.py @@ -7,6 +7,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-looker/main.py b/airbyte-integrations/connectors/source-looker/main.py index b6164cb0322bf..cb6f8adb314b0 100644 --- a/airbyte-integrations/connectors/source-looker/main.py +++ b/airbyte-integrations/connectors/source-looker/main.py @@ -4,5 +4,6 @@ from source_looker.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-looker/source_looker/components.py b/airbyte-integrations/connectors/source-looker/source_looker/components.py index d30b120a5e971..60352fffd1978 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/components.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/components.py @@ -7,14 +7,15 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth + API_VERSION = "4.0" @dataclass class LookerAuthenticator(NoAuth): - """ Authenticator that sets the Authorization header on the HTTP requests sent using access token which is updated upon expiration. diff --git a/airbyte-integrations/connectors/source-looker/source_looker/source.py b/airbyte-integrations/connectors/source-looker/source_looker/source.py index 5a6e0487f5bb0..7799272df347d 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/source.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.core import Stream + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailchimp/main.py b/airbyte-integrations/connectors/source-mailchimp/main.py index c61875fb7a72e..0145a6e026ef6 100644 --- a/airbyte-integrations/connectors/source-mailchimp/main.py +++ b/airbyte-integrations/connectors/source-mailchimp/main.py @@ -4,5 +4,6 @@ from source_mailchimp.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py index 6df59797f6aef..f853ec6a4687f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/components.py @@ -3,6 +3,7 @@ from typing import Any, Iterable, Mapping import requests + from airbyte_cdk.sources.declarative.extractors import DpathExtractor diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py index 59d0de0e98691..4952fac913eb6 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/config_migrations.py @@ -7,12 +7,14 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import FailureType from airbyte_cdk.sources import Source from airbyte_cdk.utils import AirbyteTracedException + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py index 602562aa2db46..1b37739b4ab44 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/integration/test_automations.py @@ -4,16 +4,18 @@ from unittest import TestCase import freezegun +from source_mailchimp import SourceMailchimp + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from source_mailchimp import SourceMailchimp from .config import ConfigBuilder + _CONFIG = ConfigBuilder().with_start_date(datetime.datetime(2023, 1, 1, 0, 0, 0, 1000)).build() diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py index 7f9ad40d0c7fc..ec7029480f6a7 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_component_custom_email_activity_extractor.py @@ -6,9 +6,10 @@ import json import requests -from airbyte_cdk.sources.declarative.decoders import JsonDecoder from source_mailchimp.components import MailChimpRecordExtractorEmailActivity +from airbyte_cdk.sources.declarative.decoders import JsonDecoder + def test_email_activity_extractor(): decoder = JsonDecoder(parameters={}) diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py index 20fe8312352e2..b6d0e41de2c21 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_config_datacenter_migration.py @@ -6,10 +6,12 @@ from typing import Any, Mapping import pytest -from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from source_mailchimp import SourceMailchimp from source_mailchimp.config_migrations import MigrateDataCenter +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + + # BASE ARGS SOURCE: YamlDeclarativeSource = SourceMailchimp() diff --git a/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailerlite/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailersend/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-marketo/main.py b/airbyte-integrations/connectors/source-marketo/main.py index 4b7b8e8d1708c..0b087d968a274 100644 --- a/airbyte-integrations/connectors/source-marketo/main.py +++ b/airbyte-integrations/connectors/source-marketo/main.py @@ -4,5 +4,6 @@ from source_marketo.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 418c327cc1049..1eee453d17401 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -11,6 +11,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py b/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py index 9926e2e2084e0..426d437b66a26 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/utils.py @@ -5,6 +5,7 @@ from datetime import datetime + STRING_TYPES = [ "string", "email", diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py index f088ce69b9fca..b4f03f750bd4b 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py @@ -9,9 +9,11 @@ import pendulum import pytest -from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from source_marketo.source import Activities, MarketoAuthenticator, SourceMarketo +from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream + + START_DATE = pendulum.now().subtract(days=75) @@ -77,7 +79,10 @@ def _generator(min_size: int): def fake_records_gen(): new_line = "\n" for i in range(1000): - yield f"{str(faker.random_int())},{faker.random_int()},{faker.date_of_birth()},{faker.random_int()}," f"{faker.random_int()},{faker.email()},{faker.postcode()}{new_line}" + yield ( + f"{str(faker.random_int())},{faker.random_int()},{faker.date_of_birth()},{faker.random_int()}," + f"{faker.random_int()},{faker.email()},{faker.postcode()}{new_line}" + ) size, records = 0, 0 path = os.path.realpath(str(time.time())) @@ -98,9 +103,7 @@ def fake_records_gen(): def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> DeclarativeStream: source = SourceMarketo() - matches_by_name = [ - stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name - ] + matches_by_name = [stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name] if not matches_by_name: raise ValueError("Please provide a valid stream name.") return matches_by_name[0] diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index d77625438bbc0..cb5d3988aa85e 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -11,13 +11,15 @@ import pendulum import pytest import requests +from source_marketo.source import Activities, IncrementalMarketoStream, Leads, MarketoExportCreate, MarketoStream, SourceMarketo + from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.utils import AirbyteTracedException -from source_marketo.source import Activities, IncrementalMarketoStream, Leads, MarketoExportCreate, MarketoStream, SourceMarketo from .conftest import START_DATE, get_stream_by_name + logger = logging.getLogger("airbyte") @@ -38,12 +40,7 @@ def test_should_retry_quota_exceeded(config, requests_mock): response_json = { "requestId": "d2ca#18c0b9833bf", "success": False, - "errors": [ - { - "code": "1029", - "message": "Export daily quota 500MB exceeded." - } - ] + "errors": [{"code": "1029", "message": "Export daily quota 500MB exceeded."}], } requests_mock.register_uri("GET", create_job_url, status_code=200, json=response_json) @@ -132,8 +129,8 @@ def test_activities_schema(activity, expected_schema, config): ( ( "Campaign Run ID,Choice Number,Has Predictive,Step ID,Test Variant,attributes\n" - "1,3,true,10,15,{\"spam\": \"true\"}\n" - "2,3,false,11,16,{\"spam\": \"false\"}" + '1,3,true,10,15,{"spam": "true"}\n' + '2,3,false,11,16,{"spam": "false"}' ), [ { @@ -231,9 +228,7 @@ def test_parse_response_incremental(config, requests_mock): created_at_record_1 = START_DATE.add(days=1).strftime("%Y-%m-%dT%H:%M:%SZ") created_at_record_2 = START_DATE.add(days=3).strftime("%Y-%m-%dT%H:%M:%SZ") current_state = START_DATE.add(days=2).strftime("%Y-%m-%dT%H:%M:%SZ") - response = { - "result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}] - } + response = {"result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}]} requests_mock.get("/rest/v1/campaigns.json", json=response) stream = get_stream_by_name("campaigns", config) @@ -322,16 +317,8 @@ def test_get_updated_state(config, latest_record, current_state, expected_state) def test_filter_null_bytes(config): stream = Leads(config) - test_lines = [ - "Hello\x00World\n", - "Name,Email\n", - "John\x00Doe,john.doe@example.com\n" - ] - expected_lines = [ - "HelloWorld\n", - "Name,Email\n", - "JohnDoe,john.doe@example.com\n" - ] + test_lines = ["Hello\x00World\n", "Name,Email\n", "John\x00Doe,john.doe@example.com\n"] + expected_lines = ["HelloWorld\n", "Name,Email\n", "JohnDoe,john.doe@example.com\n"] filtered_lines = stream.filter_null_bytes(test_lines) for expected_line, filtered_line in zip(expected_lines, filtered_lines): assert expected_line == filtered_line @@ -340,15 +327,8 @@ def test_filter_null_bytes(config): def test_csv_rows(config): stream = Leads(config) - test_lines = [ - "Name,Email\n", - "John Doe,john.doe@example.com\n", - "Jane Doe,jane.doe@example.com\n" - ] - expected_records = [ - {"Name": "John Doe", "Email": "john.doe@example.com"}, - {"Name": "Jane Doe", "Email": "jane.doe@example.com"} - ] + test_lines = ["Name,Email\n", "John Doe,john.doe@example.com\n", "Jane Doe,jane.doe@example.com\n"] + expected_records = [{"Name": "John Doe", "Email": "john.doe@example.com"}, {"Name": "Jane Doe", "Email": "jane.doe@example.com"}] records = stream.csv_rows(test_lines) for expected_record, record in zip(expected_records, records): assert expected_record == record diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py index 453954ab36415..38cff9225bdd1 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py @@ -8,6 +8,7 @@ import pytest from source_marketo.utils import clean_string, format_value, to_datetime_str + test_data = [ (1, {"type": "integer"}, int), ("string", {"type": "string"}, str), diff --git a/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-merge/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-metabase/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py index 88b4cf3808e82..18cbfd7178117 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/main.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/main.py @@ -4,5 +4,6 @@ from source_microsoft_dataverse.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py index a3a14fc8addbf..8f33e6eaf55d8 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/dataverse.py @@ -6,6 +6,7 @@ from typing import Any, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import Oauth2Authenticator @@ -25,7 +26,6 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: class AirbyteType(Enum): - String = {"type": ["null", "string"]} Boolean = {"type": ["null", "boolean"]} Timestamp = {"type": ["null", "string"], "format": "date-time", "airbyte_type": "timestamp_with_timezone"} @@ -34,7 +34,6 @@ class AirbyteType(Enum): class DataverseType(Enum): - String = AirbyteType.String Uniqueidentifier = AirbyteType.String DateTime = AirbyteType.Timestamp diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py index a85426a7c098f..3f384d8e31b86 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/streams.py @@ -8,13 +8,13 @@ from urllib import parse import requests + from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream # Basic full refresh stream class MicrosoftDataverseStream(HttpStream, ABC): - # Base url will be set by init(), using information provided by the user through config input url_base = "" primary_key = "" @@ -97,7 +97,6 @@ def path( # Basic incremental stream class IncrementalMicrosoftDataverseStream(MicrosoftDataverseStream, IncrementalMixin, ABC): - delta_token_field = "$deltatoken" state_checkpoint_interval = None # For now we just use the change tracking as state, and it is only emitted on last page diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py index 0e93f16521abc..402667e9e3fd6 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py @@ -6,11 +6,12 @@ from unittest import mock from unittest.mock import MagicMock -from airbyte_cdk.models import SyncMode from source_microsoft_dataverse.dataverse import AirbyteType from source_microsoft_dataverse.source import SourceMicrosoftDataverse from source_microsoft_dataverse.streams import IncrementalMicrosoftDataverseStream, MicrosoftDataverseStream +from airbyte_cdk.models import SyncMode + @mock.patch("source_microsoft_dataverse.source.do_request") def test_check_connection(mock_request): diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py index f12c7112ca116..4d33342920aa5 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py @@ -5,5 +5,6 @@ from source_microsoft_onedrive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py index b5bc8890dacce..a94c5f3d02db2 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from pydantic import BaseModel, Field +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec + class OAuthCredentials(BaseModel): """ diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py index b7c13534a3e57..2623fff2842fc 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py @@ -11,12 +11,13 @@ import requests import smart_open -from airbyte_cdk import AirbyteTracedException, FailureType -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from msal import ConfidentialClientApplication from msal.exceptions import MsalServiceError from office365.graph_client import GraphClient + +from airbyte_cdk import AirbyteTracedException, FailureType +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py index 4823f26520493..8c2d6ede5243b 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/main.py @@ -4,5 +4,6 @@ from source_microsoft_sharepoint.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py index e1053a3236967..c61196de4e475 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/spec.py @@ -6,9 +6,10 @@ from typing import Any, Dict, Literal, Optional, Union import dpath.util -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from pydantic.v1 import BaseModel, Field +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec + class OAuthCredentials(BaseModel): """ diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py index 8db55c5c66094..baab6f86505b9 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py @@ -11,11 +11,12 @@ import requests import smart_open +from msal import ConfidentialClientApplication +from office365.graph_client import GraphClient + from airbyte_cdk import AirbyteTracedException, FailureType from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile -from msal import ConfidentialClientApplication -from office365.graph_client import GraphClient from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec from .utils import FolderNotFoundException, MicrosoftSharePointRemoteFile, execute_query_with_retry, filter_http_urls diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py index 7a658c1b24319..a1741915978ba 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py @@ -8,6 +8,7 @@ from airbyte_cdk import AirbyteTracedException, FailureType from airbyte_cdk.sources.file_based.remote_file import RemoteFile + LOGGER = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py index 205e1399c52c9..ee2d8a3e78b04 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock, Mock, PropertyMock, call, patch import pytest -from airbyte_cdk import AirbyteTracedException from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec from source_microsoft_sharepoint.stream_reader import ( FileReadMode, @@ -17,6 +16,8 @@ ) from wcmatch.glob import GLOBSTAR, globmatch +from airbyte_cdk import AirbyteTracedException + def create_mock_drive_item(is_file, name, children=None): """Helper function to create a mock drive item.""" @@ -465,9 +466,10 @@ def test_get_shared_drive_object( ], ) def test_drives_property(auth_type, user_principal_name, has_refresh_token): - with patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, patch( - "source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client" - ) as mock_one_drive_client: + with ( + patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, + patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client") as mock_one_drive_client, + ): refresh_token = "dummy_refresh_token" if has_refresh_token else None # Setup for different authentication types config_mock = MagicMock( diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py index 5e88631a341fb..f2eae51b2c76b 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py @@ -6,9 +6,10 @@ from unittest.mock import Mock, patch import pytest -from airbyte_cdk import AirbyteTracedException from source_microsoft_sharepoint.utils import execute_query_with_retry, filter_http_urls +from airbyte_cdk import AirbyteTracedException + class MockResponse: def __init__(self, status_code, headers=None): @@ -49,9 +50,10 @@ def test_execute_query_with_retry(status_code, retry_after_header, expected_retr obj = Mock() obj.execute_query = Mock(side_effect=MockException(status_code, {"Retry-After": retry_after_header})) - with patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, patch( - "source_microsoft_sharepoint.utils.datetime" - ) as mock_datetime: + with ( + patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, + patch("source_microsoft_sharepoint.utils.datetime") as mock_datetime, + ): start_time = datetime(2021, 1, 1, 0, 0, 0) if retry_after_header: mock_datetime.now.side_effect = [start_time] * 2 + [ diff --git a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mixpanel/main.py b/airbyte-integrations/connectors/source-mixpanel/main.py index df8cb33fc826e..b08271eca1ab6 100644 --- a/airbyte-integrations/connectors/source-mixpanel/main.py +++ b/airbyte-integrations/connectors/source-mixpanel/main.py @@ -4,5 +4,6 @@ from source_mixpanel.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py index 757909d1f8dcf..838f3c1e17098 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/backoff_strategy.py @@ -5,6 +5,7 @@ from typing import Any, Optional, Union import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py index 1730f7673647b..9216c0cf7eaad 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/components.py @@ -6,6 +6,7 @@ import dpath.util import requests + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString @@ -32,7 +33,6 @@ def get_request_headers( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - return {"Accept": "application/json"} def get_request_params( @@ -62,7 +62,6 @@ def _request_params( return super()._request_params(stream_state, stream_slice, next_page_token, extra_params) def send_request(self, **kwargs) -> Optional[requests.Response]: - if self.reqs_per_hour_limit: if self.is_first_request: self.is_first_request = False diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py index 628cd46dcbdad..06b61176db629 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py @@ -10,6 +10,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py index a5d99f1566aed..815e24e61fa66 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/base_errors_handler.py @@ -6,6 +6,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py index 4ed3559f1af56..a47c2611854e7 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/errors_handlers/export_errors_handler.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorResolution, HttpStatusErrorHandler, ResponseAction from airbyte_protocol.models import FailureType diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 48a4184fbf426..1e5dccab2eeb7 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import pendulum + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 2d633492ddb12..c59258305abd0 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -9,11 +9,12 @@ import pendulum import requests +from pendulum import Date +from requests.auth import AuthBase + from airbyte_cdk import BackoffStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler -from pendulum import Date -from requests.auth import AuthBase from source_mixpanel.backoff_strategy import MixpanelStreamBackoffStrategy from source_mixpanel.errors_handlers import MixpanelStreamErrorHandler from source_mixpanel.utils import fix_date_time diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py index e643b5836d437..ed3d202c98e77 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py index 69c842e5e2559..1df5c3b847c12 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py @@ -45,12 +45,14 @@ def patch_time(mocker): ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH + def delete_cache_files(cache_directory): directory_path = Path(cache_directory) if directory_path.exists() and directory_path.is_dir(): for file_path in directory_path.glob("*.sqlite"): file_path.unlink() + @pytest.fixture(autouse=True) def clear_cache_before_each_test(): # The problem: Once the first request is cached, we will keep getting the cached result no matter what setup we prepared for a particular test. diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py index a2132148baa22..57da922485cae 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py @@ -5,10 +5,12 @@ from unittest.mock import patch import pytest -from airbyte_cdk.entrypoint import AirbyteEntrypoint from source_mixpanel.config_migrations import MigrateProjectId from source_mixpanel.source import SourceMixpanel +from airbyte_cdk.entrypoint import AirbyteEntrypoint + + # Test data for parametrized test test_data = [ # Test when only api_secret is present diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py index e1636caaef470..6a93ad1674713 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_property_transformation.py @@ -7,12 +7,14 @@ that will conflict in further data normalization, like: `userName` and `username` """ + from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from source_mixpanel.streams import Export +from airbyte_cdk.models import SyncMode + from .utils import get_url_to_mock, setup_response diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index f877bdd997791..4471eaa511b35 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -6,12 +6,14 @@ import logging import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel.source import SourceMixpanel, TokenAuthenticatorBase64 from source_mixpanel.streams import Export +from airbyte_cdk.utils import AirbyteTracedException + from .utils import command_check, get_url_to_mock, setup_response + logger = logging.getLogger("airbyte") @@ -22,12 +24,7 @@ def check_connection_url(config): return get_url_to_mock(export_stream) -@pytest.mark.parametrize( - "response_code,expect_success,response_json", - [ - (400, False, {"error": "Request error"}) - ] -) +@pytest.mark.parametrize("response_code,expect_success,response_json", [(400, False, {"error": "Request error"})]) def test_check_connection(requests_mock, check_connection_url, config_raw, response_code, expect_success, response_json): # requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, response_json)) requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=response_code, json=response_json) @@ -135,7 +132,7 @@ def test_streams_string_date(requests_mock, config_raw): "select_properties_by_default": True, "region": "EU", "date_window_size": 10, - "page_size": 1000 + "page_size": 1000, }, True, None, @@ -143,9 +140,9 @@ def test_streams_string_date(requests_mock, config_raw): ), ) def test_config_validation(config, success, expected_error_message, requests_mock): - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) try: is_success, message = SourceMixpanel().check_connection(None, config) except AirbyteTracedException as e: diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index 6f64dbf39aa50..9774037a6038d 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -9,15 +9,17 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel import SourceMixpanel from source_mixpanel.streams import EngageSchema, Export, ExportSchema, IncrementalMixpanelStream, MixpanelStream from source_mixpanel.utils import read_full_refresh +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.utils import AirbyteTracedException + from .utils import get_url_to_mock, read_incremental, setup_response + logger = logging.getLogger("airbyte") MIXPANEL_BASE_URL = "https://mixpanel.com/api/2.0/" @@ -95,7 +97,7 @@ def cohorts_response(): ) -def init_stream(name='', config=None): +def init_stream(name="", config=None): streams = SourceMixpanel().streams(config) for stream in streams: if stream.name == name: @@ -104,10 +106,10 @@ def init_stream(name='', config=None): def test_cohorts_stream_incremental(requests_mock, cohorts_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - config_raw['start_date'] = '2022-01-01T00:00:00Z' + config_raw["start_date"] = "2022-01-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response) - cohorts_stream = init_stream('cohorts', config=config_raw) + cohorts_stream = init_stream("cohorts", config=config_raw) records = read_incremental(cohorts_stream, stream_state={"created": "2022-04-19 23:22:01"}, cursor_field=["created"]) @@ -158,25 +160,14 @@ def engage_response(): def test_engage_stream_incremental(requests_mock, engage_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2022-02-01T00:00:00Z' - config_raw['end_date'] = '2024-05-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2022-02-01T00:00:00Z" + config_raw["end_date"] = "2024-05-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) stream_state = {"last_seen": "2024-02-11T11:20:47"} records = list(read_incremental(stream, stream_state=stream_state, cursor_field=["last_seen"])) @@ -193,97 +184,97 @@ def test_engage_stream_incremental(requests_mock, engage_response, config_raw): {}, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "abnormal_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 0, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "medium_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 1, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "early_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "state_for_different_partition", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, }, { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, - } + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, + }, ] - } + }, ), ), ) @@ -291,26 +282,17 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi """Cohort_members stream has legacy state but actually it should always return all records because members in cohorts can be updated at any time """ - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{'id': 1111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}]) + requests_mock.register_uri( + "GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{"id": 1111, "name": "bla", "created": "2024-02-02T00:00:00Z"}] + ) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('cohort_members', config=config_raw) + stream = init_stream("cohort_members", config=config_raw) records = list(read_incremental(stream, stream_state=state, cursor_field=["last_seen"])) @@ -321,99 +303,85 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi def test_cohort_members_stream_pagination(requests_mock, engage_response, config_raw): """Cohort_members pagination""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' - - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[ - {'id': 71000, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 71111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}, - {'id': 72222, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 73333, "name":'bla', 'created': '2024-02-03T00:00:00Z'}, - ]) - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage", [ - { # initial request for 71000 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - }, - { # initial request for 71111 cohort and further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 2002, - "results": [ - { - "$distinct_id": "71111_1", - "$properties": { - "$created": "2024-03-01T11:20:47", - "$last_seen": "2024-03-01T11:20:47", + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" + requests_mock.register_uri( + "GET", + MIXPANEL_BASE_URL + "cohorts/list", + json=[ + {"id": 71000, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 71111, "name": "bla", "created": "2024-02-02T00:00:00Z"}, + {"id": 72222, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 73333, "name": "bla", "created": "2024-02-03T00:00:00Z"}, + ], + ) + requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage", + [ + { # initial request for 71000 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + { # initial request for 71111 cohort and further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 2002, + "results": [ + { + "$distinct_id": "71111_1", + "$properties": { + "$created": "2024-03-01T11:20:47", + "$last_seen": "2024-03-01T11:20:47", + }, }, - }, - { - "$distinct_id": "71111_2", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", - } - } - ] - } - }, { # initial request for 72222 cohort without further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 1, - "results": [ - { - "$distinct_id": "72222_1", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", + { + "$distinct_id": "71111_2", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, + }, + ], + }, + }, + { # initial request for 72222 cohort without further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 1, + "results": [ + { + "$distinct_id": "72222_1", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, } - } - ] - } - },{ # initial request for 73333 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - } - ] + ], + }, + }, + { # initial request for 73333 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + ], ) # request for 1 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", + json={ "page": 1, "session_id": "1234567890", "status": "ok", @@ -423,13 +391,16 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) # request for 2 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", + json={ "page": 2, "session_id": "1234567890", "status": "ok", @@ -439,27 +410,23 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) - stream = init_stream('cohort_members', config=config_raw) - + stream = init_stream("cohort_members", config=config_raw) + records = list(read_incremental(stream, stream_state={}, cursor_field=["last_seen"])) assert len(records) == 5 new_updated_state = stream.get_updated_state(current_stream_state={}, latest_record=records[-1] if records else None) - assert new_updated_state == {'states': [ - { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 71111, 'parent_slice': {}} - }, - { - 'cursor': {'last_seen': '2024-02-01T11:20:47'}, - 'partition': {'id': 72222, 'parent_slice': {}} - } - ]} + assert new_updated_state == { + "states": [ + {"cursor": {"last_seen": "2024-03-01T11:20:47"}, "partition": {"id": 71111, "parent_slice": {}}}, + {"cursor": {"last_seen": "2024-02-01T11:20:47"}, "partition": {"id": 72222, "parent_slice": {}}}, + ] + } @pytest.fixture @@ -493,37 +460,30 @@ def funnels_response(start_date): }, ) + @pytest.fixture def funnel_ids_response(start_date): - return setup_response( - 200, - [{ - "funnel_id": 36152117, - "name": "test" - }] - ) + return setup_response(200, [{"funnel_id": 36152117, "name": "test"}]) def test_funnels_stream(requests_mock, config, funnels_response, funnel_ids_response, config_raw): config_raw["start_date"] = "2024-01-01T00:00:00Z" config_raw["end_date"] = "2024-04-01T00:00:00Z" - stream = init_stream('funnels', config=config_raw) + stream = init_stream("funnels", config=config_raw) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels/list", funnel_ids_response) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels", funnels_response) stream_slices = list(stream.stream_slices(sync_mode=SyncMode.incremental)) assert len(stream_slices) > 3 - assert { - "funnel_id": stream_slices[0]['funnel_id'], - "name": stream_slices[0]['funnel_name'] - } == { + assert {"funnel_id": stream_slices[0]["funnel_id"], "name": stream_slices[0]["funnel_name"]} == { "funnel_id": "36152117", - "name": "test" + "name": "test", } records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0]) records = list(records) assert len(records) == 2 + @pytest.fixture def engage_schema_response(): return setup_response( @@ -552,7 +512,7 @@ def _minimize_schema(fill_schema, schema_original): def test_engage_schema(requests_mock, engage_schema_response, config_raw): - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) requests_mock.register_uri("GET", get_url_to_mock(EngageSchema(authenticator=MagicMock(), **config_raw)), engage_schema_response) type_schema = {} _minimize_schema(type_schema, stream.get_json_schema()) @@ -600,7 +560,7 @@ def test_update_engage_schema(requests_mock, config, config_raw): }, ), ) - engage_stream = init_stream('engage', config=config_raw) + engage_stream = init_stream("engage", config=config_raw) engage_schema = engage_stream.get_json_schema() assert "someNewSchemaField" in engage_schema["properties"] @@ -619,13 +579,10 @@ def annotations_response(): def test_annotations_stream(requests_mock, annotations_response, config_raw): - stream = init_stream('annotations', config=config_raw) + stream = init_stream("annotations", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", annotations_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -648,14 +605,12 @@ def revenue_response(): "status": "ok", }, ) -def test_revenue_stream(requests_mock, revenue_response, config_raw): - stream = init_stream('revenue', config=config_raw) + +def test_revenue_stream(requests_mock, revenue_response, config_raw): + stream = init_stream("revenue", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", revenue_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -675,7 +630,6 @@ def export_schema_response(): def test_export_schema(requests_mock, export_schema_response, config): - stream = ExportSchema(authenticator=MagicMock(), **config) requests_mock.register_uri("GET", get_url_to_mock(stream), export_schema_response) @@ -684,14 +638,14 @@ def test_export_schema(requests_mock, export_schema_response, config): records_length = sum(1 for _ in records) assert records_length == 2 -def test_export_get_json_schema(requests_mock, export_schema_response, config): +def test_export_get_json_schema(requests_mock, export_schema_response, config): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", export_schema_response) stream = Export(authenticator=MagicMock(), **config) schema = stream.get_json_schema() - assert "DYNAMIC_FIELD" in schema['properties'] + assert "DYNAMIC_FIELD" in schema["properties"] @pytest.fixture @@ -717,7 +671,6 @@ def export_response(): def test_export_stream(requests_mock, export_response, config): - stream = Export(authenticator=MagicMock(), **config) requests_mock.register_uri("GET", get_url_to_mock(stream), export_response) @@ -728,8 +681,8 @@ def test_export_stream(requests_mock, export_response, config): records_length = sum(1 for _ in records) assert records_length == 1 -def test_export_stream_fail(requests_mock, export_response, config): +def test_export_stream_fail(requests_mock, export_response, config): stream = Export(authenticator=MagicMock(), **config) error_message = "" requests_mock.register_uri("GET", get_url_to_mock(stream), status_code=400, text="Unable to authenticate request") diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index 1762aa42c7185..3eeff0eac9fd2 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -9,7 +9,6 @@ def test_date_slices(): - now = pendulum.today(tz="US/Pacific").date() # test with stream_state diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py index 5b08cd7892447..4c2903b76df9b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py @@ -35,7 +35,9 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, stream_instance.state = stream_state slices = stream_instance.stream_slices(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_state=stream_state) for slice in slices: - records = stream_instance.read_records(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state) + records = stream_instance.read_records( + sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state + ) for record in records: stream_state = stream_instance.get_updated_state(stream_state, record) res.append(record) diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-monday/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-monday/main.py b/airbyte-integrations/connectors/source-monday/main.py index 14f4fa2d04399..68776ccfc1f56 100644 --- a/airbyte-integrations/connectors/source-monday/main.py +++ b/airbyte-integrations/connectors/source-monday/main.py @@ -4,5 +4,6 @@ from source_monday.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-monday/source_monday/components.py b/airbyte-integrations/connectors/source-monday/source_monday/components.py index 802d23aacc811..d79e8c8e01245 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/components.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/components.py @@ -6,6 +6,7 @@ from typing import Any, Iterable, List, Mapping, Optional, Union import dpath.util + from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.incremental import Cursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -14,6 +15,7 @@ from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.core import Stream + RequestInput = Union[str, Mapping[str, str]] diff --git a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py index 126839bdecc70..6b0d08427d935 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py @@ -10,12 +10,14 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config, Record + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py index a6276416d2e5d..131f69786a8ec 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement + # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-monday/source_monday/source.py b/airbyte-integrations/connectors/source-monday/source_monday/source.py index 45fab7ff246ff..868e33c0a0279 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/source.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py index 3dd017d476b78..b25b2c27ad98f 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class MondayBaseRequestBuilder(MondayRequestBuilder): diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py index 779d64d80af7c..2457b51faac00 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py @@ -19,4 +19,3 @@ def build(self, file_path: Optional[str] = None) -> HttpResponse: if not file_path: return HttpResponse(json.dumps(find_template(str(self._status_code), __file__)), self._status_code) return HttpResponse(json.dumps(find_template(str(file_path), __file__)), self._status_code) - diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py index eab0deb8d5b1a..654e84f751928 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/utils.py @@ -3,12 +3,13 @@ import operator from typing import Any, Dict, List, Optional +from source_monday import SourceMonday + from airbyte_cdk.models import AirbyteMessage from airbyte_cdk.models import Level as LogLevel from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import SyncMode -from source_monday import SourceMonday def read_stream( diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py index 30571cbd43f7b..4b0aab530fb46 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py @@ -7,13 +7,14 @@ from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.models import AirbyteMessage, SyncMode, Type -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.streams import Stream from requests import Response from source_monday.components import IncrementalSingleSlice, IncrementalSubstreamSlicer from source_monday.extractor import MondayIncrementalItemsExtractor +from airbyte_cdk.models import AirbyteMessage, SyncMode, Type +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.streams import Stream + def _create_response(content: Any) -> Response: response = Response() @@ -144,7 +145,6 @@ def mock_parent_stream_slices(*args, **kwargs): ids=["no stream state", "successfully read parent record", "skip non_record AirbyteMessage"], ) def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, expected_slices): - slicer = IncrementalSubstreamSlicer( config={}, parameters={}, @@ -162,7 +162,6 @@ def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, ex def test_set_initial_state(): - slicer = IncrementalSubstreamSlicer( config={}, parameters={}, diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index 2037f13ee02a1..fcd494ecb348e 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -5,10 +5,12 @@ from unittest.mock import MagicMock import pytest +from source_monday import MondayGraphqlRequester + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader -from source_monday import MondayGraphqlRequester + nested_object_schema = { "root": { @@ -126,7 +128,6 @@ def test_get_schema_root_properties(mocker, monday_requester): def test_build_activity_query(mocker, monday_requester): - mock_stream_state = {"updated_at_int": 1636738688} object_arguments = {"stream_state": mock_stream_state} mocker.patch.object(MondayGraphqlRequester, "_get_object_arguments", return_value="stream_state:{{ stream_state['updated_at_int'] }}") @@ -140,7 +141,6 @@ def test_build_activity_query(mocker, monday_requester): def test_build_items_incremental_query(monday_requester): - object_name = "test_items" field_schema = { "id": {"type": "integer"}, @@ -151,20 +151,21 @@ def test_build_items_incremental_query(monday_requester): "text": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}, - "display_value": {"type": ["null", "string"]} + "display_value": {"type": ["null", "string"]}, } - } + }, } stream_slice = {"ids": [1, 2, 3]} built_query = monday_requester._build_items_incremental_query(object_name, field_schema, stream_slice) - assert built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," \ - "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + assert ( + built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," + "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + ) def test_get_request_headers(monday_requester): - headers = monday_requester.get_request_headers() assert headers == {"API-Version": "2024-01"} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mssql/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-my-hours/components.py b/airbyte-integrations/connectors/source-my-hours/components.py index 4377ee24ff37c..109b493ebd030 100644 --- a/airbyte-integrations/connectors/source-my-hours/components.py +++ b/airbyte-integrations/connectors/source-my-hours/components.py @@ -7,10 +7,12 @@ from typing import Any, Mapping, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://docs.airbyte.com/integrations/sources/my-hours # The Bearer token generated will expire in five days diff --git a/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-my-hours/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py index 488bcf605f2ec..0fcfbd6ff847d 100755 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py @@ -14,6 +14,7 @@ import pytz from mysql.connector import Error + support_file_path_prefix = "/connector/integration_tests" catalog_write_file = support_file_path_prefix + "/temp/configured_catalog_copy.json" catalog_source_file = support_file_path_prefix + "/configured_catalog_template.json" @@ -22,12 +23,13 @@ abnormal_state_write_file = support_file_path_prefix + "/temp/abnormal_state_copy.json" abnormal_state_file = support_file_path_prefix + "/abnormal_state_template.json" -secret_config_file = '/connector/secrets/cat-config.json' -secret_active_config_file = support_file_path_prefix + '/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/cat-config-cdc.json' -secret_active_config_cdc_file = support_file_path_prefix + '/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/cat-config.json" +secret_active_config_file = support_file_path_prefix + "/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/cat-config-cdc.json" +secret_active_config_cdc_file = support_file_path_prefix + "/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') @contextmanager def connect_to_db(): @@ -36,11 +38,7 @@ def connect_to_db(): conn = None try: conn = mysql.connector.connect( - database=None, - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + database=None, user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") yield conn @@ -54,6 +52,7 @@ def connect_to_db(): conn.close() print("Database connection closed") + def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: insert_query = f"INSERT INTO {schema_name}.{table_name} (id, name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE id=id" try: @@ -66,6 +65,7 @@ def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[ print(f"Error inserting records: {error}") conn.rollback() + def create_schema(conn, schema_name: str) -> None: create_schema_query = f"CREATE DATABASE IF NOT EXISTS {schema_name}" try: @@ -77,32 +77,34 @@ def create_schema(conn, schema_name: str) -> None: print(f"Error creating database: {error}") conn.rollback() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path(support_file_path_prefix + "/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn, schema_name: str, table_name: str) -> None: create_table_query = f""" CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( @@ -119,45 +121,44 @@ def create_table(conn, schema_name: str, table_name: str) -> None: print(f"Error creating table: {error}") conn.rollback() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] - table_name = 'id_and_name_cat' + new_records = [("4", "four"), ("5", "five")] + table_name = "id_and_name_cat" with connect_to_db() as conn: insert_records(conn, schema_name, table_name, new_records) + def setup(): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] with connect_to_db() as conn: create_schema(conn, schema_name) create_table(conn, schema_name, table_name) insert_records(conn, schema_name, table_name, records) + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_schemas_with_prefix(conn, date_prefix): query = f""" SELECT schema_name @@ -177,19 +178,22 @@ def delete_schemas_with_prefix(conn, date_prefix): print(f"An error occurred in deleting schema: {e}") sys.exit(1) + def teardown() -> None: today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") with connect_to_db() as conn: delete_schemas_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: schema_name = load_schema_name_from_catalog() print(f"delete database {schema_name}") with connect_to_db() as conn: delete_schemas_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-n8n/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py index ea1ca1161ee24..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-netsuite/integration_tests/acceptance.py @@ -5,10 +5,10 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) @pytest.fixture(scope="session", autouse=True) def connector_setup(): - yield diff --git a/airbyte-integrations/connectors/source-netsuite/main.py b/airbyte-integrations/connectors/source-netsuite/main.py index 492266da15e29..5d7d745b82af6 100644 --- a/airbyte-integrations/connectors/source-netsuite/main.py +++ b/airbyte-integrations/connectors/source-netsuite/main.py @@ -4,5 +4,6 @@ from source_netsuite.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-netsuite/setup.py b/airbyte-integrations/connectors/source-netsuite/setup.py index e16d4d5b270f3..682252ab21246 100644 --- a/airbyte-integrations/connectors/source-netsuite/setup.py +++ b/airbyte-integrations/connectors/source-netsuite/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", "requests-oauthlib", diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py index 610adece49443..e7f64474b9b23 100644 --- a/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/source.py @@ -9,15 +9,15 @@ from typing import Any, List, Mapping, Tuple, Union import requests +from requests_oauthlib import OAuth1 + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from requests_oauthlib import OAuth1 from source_netsuite.constraints import CUSTOM_INCREMENTAL_CURSOR, INCREMENTAL_CURSOR, META_PATH, RECORD_PATH, SCHEMA_HEADERS from source_netsuite.streams import CustomIncrementalNetsuiteStream, IncrementalNetsuiteStream, NetsuiteStream class SourceNetsuite(AbstractSource): - logger: logging.Logger = logging.getLogger("airbyte") def auth(self, config: Mapping[str, Any]) -> OAuth1: @@ -109,7 +109,6 @@ def generate_stream( window_in_days: int, max_retry: int = 3, ) -> Union[NetsuiteStream, IncrementalNetsuiteStream, CustomIncrementalNetsuiteStream]: - input_args = { "auth": auth, "object_name": object_name, diff --git a/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py b/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py index 57ab516437829..4d59ac5470343 100644 --- a/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py +++ b/airbyte-integrations/connectors/source-netsuite/source_netsuite/streams.py @@ -9,8 +9,9 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union import requests -from airbyte_cdk.sources.streams.http import HttpStream from requests_oauthlib import OAuth1 + +from airbyte_cdk.sources.streams.http import HttpStream from source_netsuite.constraints import ( CUSTOM_INCREMENTAL_CURSOR, INCREMENTAL_CURSOR, @@ -159,7 +160,6 @@ def parse_response( next_page_token: Mapping[str, Any] = None, **kwargs, ) -> Iterable[Mapping]: - records = response.json().get("items") request_kwargs = self.request_kwargs(stream_slice, next_page_token) if records: diff --git a/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-news-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-newsdata/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-notion/main.py b/airbyte-integrations/connectors/source-notion/main.py index 671d6cd692faa..0c88cd7df7d34 100644 --- a/airbyte-integrations/connectors/source-notion/main.py +++ b/airbyte-integrations/connectors/source-notion/main.py @@ -4,5 +4,6 @@ from source_notion.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 5b92270c10e26..d93ce9fb56d2d 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -9,13 +9,15 @@ import pendulum import pydantic import requests +from requests import HTTPError + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import CheckpointMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException -from requests import HTTPError + # maximum block hierarchy recursive request depth MAX_BLOCK_DEPTH = 30 @@ -27,7 +29,6 @@ class NotionAvailabilityStrategy(HttpAvailabilityStrategy): """ def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: - reasons_for_codes: Dict[int, str] = { requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " "Please make sure your integration has read access for the resources you are trying to sync" @@ -36,7 +37,6 @@ def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, s class NotionStream(HttpStream, ABC): - url_base = "https://api.notion.com/v1/" primary_key = "id" @@ -146,7 +146,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class StateValueWrapper(pydantic.BaseModel): - stream: T state_value: str max_cursor_time: Any = "" @@ -166,7 +165,6 @@ def dict(self, **kwargs): class IncrementalNotionStream(NotionStream, CheckpointMixin, ABC): - cursor_field = "last_edited_time" http_method = "POST" diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py index 6f59654861817..be48cadc1a219 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py @@ -8,52 +8,79 @@ def test_users_stream_transformation(): input_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "user": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } output_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "info": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } assert NotionUserTransformation().transform(input_record) == output_record def test_notion_properties_transformation(): input_record = { - "id": "123", "properties": { - "Due date": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - }, + "id": "123", + "properties": { + "Due date": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, "Status": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, } output_record = { - "id": "123", "properties": [ + "id": "123", + "properties": [ { - "name": "Due date", "value": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - } + "name": "Due date", + "value": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, }, { "name": "Status", "value": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } - ] + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, + ], } assert NotionPropertiesTransformation().transform(input_record) == output_record @@ -64,55 +91,38 @@ def test_notion_properties_transformation(): {"id": "3", "last_edited_time": "2022-01-04T00:00:00.000Z"}, ] + @pytest.fixture def data_feed_config(): return NotionDataFeedFilter(parameters={}, config={"start_date": "2021-01-01T00:00:00.000Z"}) + @pytest.mark.parametrize( "state_value, expected_return", [ - ( - "2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z" - ), - ( - "2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z" - ), - ( - {}, "2021-01-01T00:00:00.000Z" - ) + ("2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z"), + ("2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z"), + ({}, "2021-01-01T00:00:00.000Z"), ], - ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"] + ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"], ) def test_data_feed_get_filter_date(data_feed_config, state_value, expected_return): start_date = data_feed_config.config["start_date"] - + result = data_feed_config._get_filter_date(start_date, state_value) assert result == expected_return, f"Expected {expected_return}, but got {result}." -@pytest.mark.parametrize("stream_state,stream_slice,expected_records", [ - ( - {"last_edited_time": "2022-01-01T00:00:00.000Z"}, - {"id": "some_id"}, - state_test_records - ), - ( - {"last_edited_time": "2022-01-03T00:00:00.000Z"}, - {"id": "some_id"}, - [state_test_records[-2], state_test_records[-1]] - ), - ( - {"last_edited_time": "2022-01-05T00:00:00.000Z"}, - {"id": "some_id"}, - [] - ), - ( - {}, - {"id": "some_id"}, - state_test_records - ) -], -ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"]) +@pytest.mark.parametrize( + "stream_state,stream_slice,expected_records", + [ + ({"last_edited_time": "2022-01-01T00:00:00.000Z"}, {"id": "some_id"}, state_test_records), + ({"last_edited_time": "2022-01-03T00:00:00.000Z"}, {"id": "some_id"}, [state_test_records[-2], state_test_records[-1]]), + ({"last_edited_time": "2022-01-05T00:00:00.000Z"}, {"id": "some_id"}, []), + ({}, {"id": "some_id"}, state_test_records), + ], + ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"], +) def test_data_feed_filter_records(data_feed_config, stream_state, stream_slice, expected_records): filtered_records = data_feed_config.filter_records(state_test_records, stream_state, stream_slice) assert filtered_records == expected_records, "Filtered records do not match the expected records." diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py index 7b323ed72257b..46b97f977cb25 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py @@ -10,11 +10,12 @@ import freezegun import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException from pytest import fixture, mark from source_notion.streams import Blocks, IncrementalNotionStream, NotionStream, Pages +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException + @pytest.fixture def patch_base_class(mocker): @@ -79,11 +80,7 @@ def test_http_method(patch_base_class): @pytest.mark.parametrize( "response_json, expected_output", - [ - ({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), - ({"has_more": False}, None), - ({}, None) - ], + [({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), ({"has_more": False}, None), ({}, None)], ids=["Next_page_token exists with cursor", "No next_page_token", "No next_page_token"], ) def test_next_page_token(patch_base_class, response_json, expected_output): @@ -454,21 +451,149 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, def test_block_record_transformation(): stream = Blocks(parent=None, config=MagicMock()) response_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, - "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "page": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } expected_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], - "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "info": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } assert stream.transform(response_record) == expected_record diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py index c270f0894e5e4..3a5696e775749 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_notion.source import SourceNotion +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.mark.parametrize( "config, expected_token", @@ -29,8 +30,7 @@ def test_get_authenticator(config, expected_token): def test_streams(): source = SourceNotion() - config_mock = {"start_date": "2020-01-01T00:00:00.000Z", - "credentials": {"auth_type": "token", "token": "abcd"}} + config_mock = {"start_date": "2020-01-01T00:00:00.000Z", "credentials": {"auth_type": "token", "token": "abcd"}} streams = source.streams(config_mock) expected_streams_number = 5 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-nytimes/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py index 6f2ccc74201af..6541b9a7db2cc 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-okta/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-okta/main.py b/airbyte-integrations/connectors/source-okta/main.py index 7b372b3507e75..dabf89ea556f6 100644 --- a/airbyte-integrations/connectors/source-okta/main.py +++ b/airbyte-integrations/connectors/source-okta/main.py @@ -4,5 +4,6 @@ from source_okta.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-okta/source_okta/components.py b/airbyte-integrations/connectors/source-okta/source_okta/components.py index bfe8588781975..1f95fcaba7ad6 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/components.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/components.py @@ -9,6 +9,7 @@ import jwt import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.types import Config diff --git a/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py b/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py index 82fcef527e4b0..eb7e58a41d68d 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/config_migration.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index b550116424ccb..ed2e1e6ebc47d 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py index 0211ef7e482f2..9cf815ff3ddaf 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_migration.py @@ -5,11 +5,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_okta.config_migration import OktaConfigMigration from source_okta.source import SourceOkta +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + CMD = "check" SOURCE: Source = SourceOkta() diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py index 546e50c590419..9b880749910f8 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py @@ -9,10 +9,11 @@ import pytest import requests -from airbyte_cdk.sources.streams import Stream from source_okta.components import CustomBearerAuthenticator, CustomOauth2Authenticator from source_okta.source import SourceOkta +from airbyte_cdk.sources.streams import Stream + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceOkta() diff --git a/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-omnisend/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-oracle/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-orb/components.py b/airbyte-integrations/connectors/source-orb/components.py index 3732d1f5954ac..cf349c3949365 100644 --- a/airbyte-integrations/connectors/source-orb/components.py +++ b/airbyte-integrations/connectors/source-orb/components.py @@ -23,7 +23,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Record: - # for each top level response record, there can be multiple sub-records depending # on granularity and other input params. This function yields one transformed record # for each subrecord in the response. @@ -63,7 +62,6 @@ class SubscriptionUsagePartitionRouter(StreamSlicer): config: Config def stream_slices(self) -> Iterable[StreamSlice]: - """ This stream is sliced per `subscription_id` and day, as well as `billable_metric_id` if a grouping key is provided. This is because the API only supports a diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-orb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-oura/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/main.py b/airbyte-integrations/connectors/source-outbrain-amplify/main.py index dc58d39d5bcd2..4b31ffa82fcf4 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/main.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/main.py @@ -4,5 +4,6 @@ from source_outbrain_amplify.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py index 18248b027d466..4fa2955b1a756 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py @@ -5,9 +5,10 @@ from typing import Any, Mapping import requests -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from requests.auth import HTTPBasicAuth +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + class OutbrainAmplifyAuthenticator(TokenAuthenticator): def __init__(self, config, url_base): diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py index ae0a7b4d6c706..628314fc46591 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -15,6 +16,7 @@ from .auth import OutbrainAmplifyAuthenticator + DEFAULT_END_DATE = pendulum.now() DEFAULT_GEO_LOCATION_BREAKDOWN = "region" DEFAULT_REPORT_GRANULARITY = "daily" @@ -122,7 +124,6 @@ def name(self) -> str: def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -174,7 +175,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -223,7 +223,6 @@ def name(self) -> str: def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -267,7 +266,6 @@ def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -323,7 +321,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -376,7 +373,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -437,7 +433,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -500,7 +495,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -569,7 +563,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -635,7 +628,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -696,7 +688,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -759,7 +750,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -820,7 +810,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -883,7 +872,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -946,7 +934,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -1011,7 +998,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def stream_slices( self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state ) @@ -1051,7 +1037,6 @@ def path( class IncrementalOutbrainAmplifyStream(OutbrainAmplifyStream, ABC): - state_checkpoint_interval = None @property @@ -1098,7 +1083,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # Budget for Marketers stream. # 1. Budget stream based on marketers id. - stream.extend([BudgetsForMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config))]), + (stream.extend([BudgetsForMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config))]),) # Promoted Links stream. # 1. Promoted Links stream for campaigns. diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py index 5f06ad6891b36..f545aa24ff739 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_outbrain_amplify.source import IncrementalOutbrainAmplifyStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-outreach/main.py b/airbyte-integrations/connectors/source-outreach/main.py index a5b35a64025f6..3fbcc84e8fb75 100644 --- a/airbyte-integrations/connectors/source-outreach/main.py +++ b/airbyte-integrations/connectors/source-outreach/main.py @@ -4,5 +4,6 @@ from source_outreach.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/components.py b/airbyte-integrations/connectors/source-outreach/source_outreach/components.py index 48ee70c5c1b6d..2fe67a2a6bbd8 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/components.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/components.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py index f9daaa87c1e44..a3af865c1b7f8 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/run.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_outreach import SourceOutreach +from airbyte_cdk.entrypoint import launch + def run(): source = SourceOutreach() diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 3faed388f99ec..4f55b84029a8e 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pagerduty/main.py b/airbyte-integrations/connectors/source-pagerduty/main.py index 22537946cc8c7..15c7eed00df9c 100644 --- a/airbyte-integrations/connectors/source-pagerduty/main.py +++ b/airbyte-integrations/connectors/source-pagerduty/main.py @@ -4,5 +4,6 @@ from source_pagerduty.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py index f73afe9c65927..21f3c24f9f925 100644 --- a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_pagerduty import SourcePagerduty +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePagerduty() diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py index 96ff2d87bbfbc..2008ace84e363 100644 --- a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pardot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-partnerstack/main.py b/airbyte-integrations/connectors/source-partnerstack/main.py index d22642a3ee660..6cc90fad696de 100644 --- a/airbyte-integrations/connectors/source-partnerstack/main.py +++ b/airbyte-integrations/connectors/source-partnerstack/main.py @@ -4,5 +4,6 @@ from source_partnerstack.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py index 1b7dad130b551..9fc7e58ecb5f0 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_partnerstack import SourcePartnerstack +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePartnerstack() diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py index 79d39b4f690be..baacf122c6e73 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py index c371024b7ff94..02aa30582dd49 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/disputes_generator.py @@ -62,7 +62,6 @@ def read_json(filepath): def main(): - operation = sys.argv[1] CREDS = read_json("../secrets/config.json") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py index cf9d8d86e92dd..2228f436d5f54 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/fixture_helper.py @@ -8,6 +8,7 @@ # %% import requests + logging.basicConfig(level=logging.DEBUG) # %% diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py index 6a2f46c3b5240..8be90ea420c7a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/payments_generator.py @@ -82,7 +82,6 @@ def read_json(filepath): def main(): - CREDS = read_json("../secrets/config.json") client_id = CREDS.get("client_id") secret_id = CREDS.get("client_secret") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py index d8067ec1e2bf6..1568d1ace9b36 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/bin/paypal_transaction_generator.py @@ -34,6 +34,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait + # from pprint import pprint @@ -104,7 +105,6 @@ def read_json(filepath): def get_api_token(): - client_id = CREDS.get("client_id") secret = CREDS.get("client_secret") @@ -126,7 +126,6 @@ def random_digits(digits): def make_payment(): - # generate new invoice_number PAYMENT_DATA["transactions"][0]["invoice_number"] = random_digits(11) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 06823a4a71e50..302035ec0bc28 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -4,5 +4,6 @@ from source_paypal_transaction.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py index af883e9c1c19c..814bfd5b0c434 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py @@ -10,11 +10,13 @@ import backoff import requests + from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py index 1a6d4cc56c0ed..a6433dd29bdfa 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_paypal_transaction import SourcePaypalTransaction +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePaypalTransaction() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 07d75f4bc2811..c71a7fb2a7823 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -6,6 +6,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. @@ -13,6 +14,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourcePaypalTransaction(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py index dd19b6306e770..1bf0f5692aaae 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py @@ -7,58 +7,59 @@ import pytest import requests import requests_mock -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from source_paypal_transaction.components import PayPalOauth2Authenticator +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + @pytest.fixture def mock_authenticator(): return PayPalOauth2Authenticator( config={}, parameters={}, - client_id='test_client_id', - client_secret='test_client_secret', - token_refresh_endpoint='https://test.token.endpoint', - grant_type='test_grant_type' + client_id="test_client_id", + client_secret="test_client_secret", + token_refresh_endpoint="https://test.token.endpoint", + grant_type="test_grant_type", ) + def test_get_refresh_access_token_response(mock_authenticator): - expected_response_json = {'access_token': 'test_access_token', 'expires_in': 3600} + expected_response_json = {"access_token": "test_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=expected_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=expected_response_json, status_code=200) # Call _get_refresh method mock_authenticator._get_refresh_access_token_response() - - assert mock_authenticator.access_token == expected_response_json['access_token'] + + assert mock_authenticator.access_token == expected_response_json["access_token"] + def test_token_expiration(mock_authenticator): # Mock response for initial token request - initial_response_json = {'access_token': 'initial_access_token', 'expires_in': 1} + initial_response_json = {"access_token": "initial_access_token", "expires_in": 1} # Mock response for token refresh request - refresh_response_json = {'access_token': 'refreshed_access_token', 'expires_in': 3600} + refresh_response_json = {"access_token": "refreshed_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - - mock_request.post('https://test.token.endpoint', json=initial_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=initial_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the initial access token is set correctly - assert mock_authenticator.access_token == initial_response_json['access_token'] + assert mock_authenticator.access_token == initial_response_json["access_token"] time.sleep(2) - mock_request.post('https://test.token.endpoint', json=refresh_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=refresh_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the access token is refreshed - assert mock_authenticator.access_token == refresh_response_json['access_token'] + assert mock_authenticator.access_token == refresh_response_json["access_token"] def test_backoff_retry(mock_authenticator, caplog): - - mock_response = {'access_token': 'test_access_token', 'expires_in': 3600} + mock_response = {"access_token": "test_access_token", "expires_in": 3600} mock_reason = "Too Many Requests" - + with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=mock_response, status_code=429, reason=mock_reason) + mock_request.post("https://test.token.endpoint", json=mock_response, status_code=429, reason=mock_reason) with caplog.at_level(logging.INFO): try: mock_authenticator._get_refresh_access_token_response() @@ -67,6 +68,7 @@ def test_backoff_retry(mock_authenticator, caplog): else: pytest.fail("Expected DefaultBackoffException to be raised") + @pytest.fixture def authenticator_parameters(): return { @@ -75,14 +77,12 @@ def authenticator_parameters(): "config": {}, "parameters": {}, "token_refresh_endpoint": "https://test.token.endpoint", - "grant_type": "test_grant_type" + "grant_type": "test_grant_type", } + def test_get_headers(authenticator_parameters): expected_basic_auth = "Basic dGVzdF9jbGllbnRfaWQ6dGVzdF9jbGllbnRfc2VjcmV0" authenticator = PayPalOauth2Authenticator(**authenticator_parameters) headers = authenticator.get_headers() assert headers == {"Authorization": expected_basic_auth} - - - diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index 06dd08dc74a6d..bd2f9c89836a8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -11,21 +11,21 @@ @pytest.fixture(name="config") def config_fixture(): - #From File test + # From File test # with open('../secrets/config.json') as f: # return json.load(f) - #Mock test + # Mock test return { - "client_id": "your_client_id", - "client_secret": "your_client_secret", - "start_date": "2024-01-30T00:00:00Z", - "end_date": "2024-02-01T00:00:00Z", - "dispute_start_date": "2024-02-01T00:00:00.000Z", - "dispute_end_date": "2024-02-05T23:59:00.000Z", - "buyer_username": "Your Buyer email", - "buyer_password": "Your Buyer Password", - "payer_id": "ypur ACCOUNT ID", - "is_sandbox": True + "client_id": "your_client_id", + "client_secret": "your_client_secret", + "start_date": "2024-01-30T00:00:00Z", + "end_date": "2024-02-01T00:00:00Z", + "dispute_start_date": "2024-02-01T00:00:00.000Z", + "dispute_end_date": "2024-02-05T23:59:00.000Z", + "buyer_username": "Your Buyer email", + "buyer_password": "Your Buyer Password", + "payer_id": "ypur ACCOUNT ID", + "is_sandbox": True, } @@ -33,21 +33,24 @@ def config_fixture(): def source_fixture(): return SourcePaypalTransaction() + def validate_date_format(date_str, format): try: datetime.strptime(date_str, format) return True except ValueError: return False - + + def test_date_formats_in_config(config): start_date_format = "%Y-%m-%dT%H:%M:%SZ" dispute_date_format = "%Y-%m-%dT%H:%M:%S.%fZ" - assert validate_date_format(config['start_date'], start_date_format), "Start date format is incorrect" - assert validate_date_format(config['end_date'], start_date_format), "End date format is incorrect" - assert validate_date_format(config['dispute_start_date'], dispute_date_format), "Dispute start date format is incorrect" - assert validate_date_format(config['dispute_end_date'], dispute_date_format), "Dispute end date format is incorrect" + assert validate_date_format(config["start_date"], start_date_format), "Start date format is incorrect" + assert validate_date_format(config["end_date"], start_date_format), "End date format is incorrect" + assert validate_date_format(config["dispute_start_date"], dispute_date_format), "Dispute start date format is incorrect" + assert validate_date_format(config["dispute_end_date"], dispute_date_format), "Dispute end date format is incorrect" + @pytest.fixture(name="logger_mock") def logger_mock_fixture(): - return patch("source_paypal_transactions.source.AirbyteLogger") \ No newline at end of file + return patch("source_paypal_transactions.source.AirbyteLogger") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py index 958db41262da2..ddf832641c2d6 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py @@ -8,6 +8,7 @@ import pytest import requests import requests_mock + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean @@ -27,23 +28,23 @@ class CursorPaginationStrategy(PaginationStrategy): stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating decoder (Decoder): decoder to decode the response """ + cursor_value: Union[InterpolatedString, str] config: Config parameters: Mapping[str, Any] page_size: Optional[int] = None stop_condition: Optional[Union[InterpolatedBoolean, str]] = None decoder: Decoder = field(default_factory=JsonDecoder) - + def __post_init__(self): if isinstance(self.cursor_value, str): self.cursor_value = InterpolatedString.create(self.cursor_value, parameters=self.parameters) if isinstance(self.stop_condition, str): self.stop_condition = InterpolatedBoolean(condition=self.stop_condition, parameters=self.parameters) - + @property def initial_token(self) -> Optional[Any]: return None - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) @@ -51,53 +52,45 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin headers["link"] = response.links print("STOP CONDITION", self.stop_condition) - + if self.stop_condition: should_stop = self.stop_condition.eval(self.config, response=decoded_response, headers=headers, last_records=last_records) if should_stop: print("Stopping...") return None - + # Update cursor_value with the next_id from the response self.cursor_value = InterpolatedString.create(decoded_response.get("next_id"), parameters=self.parameters) token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response, headers=headers) print("TOKEN", token) return token if token else None - + def reset(self): pass - + def get_page_size(self) -> Optional[int]: return self.page_size @pytest.fixture def mock_responses(): - return [ - "token_page_init.json", - "token_PAY-0L38757939422510JMW5ZJVA.json", - "token_PAYID-MW5XXZY5YL87592N34454913.json" - ] + return ["token_page_init.json", "token_PAY-0L38757939422510JMW5ZJVA.json", "token_PAYID-MW5XXZY5YL87592N34454913.json"] + @pytest.fixture -def cursor_pagination_strategy(mock_responses, stop_condition = None): +def cursor_pagination_strategy(mock_responses, stop_condition=None): parameters = {} decoder = JsonDecoder(parameters=parameters) cursor_value = "start_id" # Initialize with a default value - + for response_file in mock_responses: if cursor_value == "start_id": cursor_value = load_mock_data(response_file).get("next_id") else: break # Stop after getting the next_id from the first response - + return CursorPaginationStrategy( - cursor_value=cursor_value, - config={}, - parameters=parameters, - page_size=3, - stop_condition=stop_condition, - decoder=decoder + cursor_value=cursor_value, config={}, parameters=parameters, page_size=3, stop_condition=stop_condition, decoder=decoder ) @@ -105,6 +98,7 @@ def load_mock_data(filename): with open(os.path.join("./unit_tests/test_files", filename), "r") as file: return json.load(file) + def test_cursor_pagination(cursor_pagination_strategy, mock_responses): with requests_mock.Mocker() as m: base_url = "http://example.com/api/resource" @@ -126,21 +120,21 @@ def test_cursor_pagination(cursor_pagination_strategy, mock_responses): if i < len(mock_responses) - 1: next_id = load_mock_data(response_file)["next_id"] print("FOUND NEXT ID:", next_id) - + else: next_id = None - cursor_pagination_strategy(mock_responses, stop_condition = True) + cursor_pagination_strategy(mock_responses, stop_condition=True) # Make API call and process response response = requests.get(url) print("GET RESPONSE:", response) assert response.status_code == 200 - + decoded_response = response.json() last_records = decoded_response["payments"] next_id = cursor_pagination_strategy.next_page_token(response, last_records) print("NEXT ID:", next_id) - + # Verify the pagination stopped assert next_id is None print("No more pages") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py index 05b98d04f90a1..1166076c79724 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py @@ -6,6 +6,7 @@ import pytest import requests import requests_mock + from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy @@ -29,24 +30,25 @@ def reset(self): def get_page_size(self): return self.page_size + @pytest.fixture def mock_pagination_strategy(): return MockPaginationStrategy(page_size=500) + @pytest.fixture def paginator(): pagination_strategy = MockPaginationStrategy(page_size=3) return DefaultPaginator( - pagination_strategy=pagination_strategy, - config={}, - url_base="http://example.com/v1/reporting/transactions", - parameters={} + pagination_strategy=pagination_strategy, config={}, url_base="http://example.com/v1/reporting/transactions", parameters={} ) - + + def load_mock_data(page): with open(f"./unit_tests/test_files/page_{page}.json", "r") as file: return file.read() + # Test to verify pagination logic transitions from page 1 to page 2 def test_pagination_logic(paginator): page_1_data = load_mock_data(1) @@ -54,7 +56,7 @@ def test_pagination_logic(paginator): paginator_url_1 = f"{paginator.url_base.string}?page=1&page_size={paginator.pagination_strategy.get_page_size}" paginator_url_2 = f"{paginator.url_base.string}?page=2&page_size={paginator.pagination_strategy.get_page_size}" - + with requests_mock.Mocker() as m: m.get(paginator_url_1, text=page_1_data, status_code=200) m.get(paginator_url_2, text=page_2_data, status_code=200) @@ -64,16 +66,14 @@ def test_pagination_logic(paginator): response_page_2 = requests.get(paginator_url_2) response_page_2._content = str.encode(page_2_data) - # Simulate getting the next page token from page 1's response next_page_token_page_1 = paginator.next_page_token(response_page_1, []) print("NEXT PAGE TOKEN", next_page_token_page_1) # Assert that the next page token indicates moving to page 2 - assert next_page_token_page_1['next_page_token'] == 2, "Failed to transition from page 1 to page 2" + assert next_page_token_page_1["next_page_token"] == 2, "Failed to transition from page 1 to page 2" - # Check that the correct page size is used in requests and that we have the right number of pages - assert len(m.request_history) == 2 - assert "page_size=3" in m.request_history[0].url - assert "page_size=3" in m.request_history[1].url \ No newline at end of file + assert len(m.request_history) == 2 + assert "page_size=3" in m.request_history[0].url + assert "page_size=3" in m.request_history[1].url diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pendo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pexels-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pinterest/main.py b/airbyte-integrations/connectors/source-pinterest/main.py index aff013c703198..59806e3eb5851 100644 --- a/airbyte-integrations/connectors/source-pinterest/main.py +++ b/airbyte-integrations/connectors/source-pinterest/main.py @@ -4,5 +4,6 @@ from source_pinterest.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py index f31a115c09a9d..91a946b0b7110 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/components/auth.py @@ -9,12 +9,14 @@ import backoff import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.airbyte_secrets_utils import add_to_secrets + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py index d689a4643205c..bfdcd32d582b5 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/python_stream_auth.py @@ -6,6 +6,7 @@ import pendulum import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py index 892b5fc652f21..f469def56e9cc 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -8,9 +8,10 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional from urllib.parse import urljoin -import airbyte_cdk.sources.utils.casing as casing import backoff import requests + +import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index bdcb8f7c00cec..44de3103bdc75 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping import pendulum + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream @@ -32,6 +33,7 @@ ) from .streams import PinterestStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py index d90f3e0862e64..94045a7e7013a 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -9,6 +9,7 @@ import pendulum import requests + from airbyte_cdk import AirbyteTracedException, BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import WaitTimeFromHeaderBackoffStrategy @@ -20,6 +21,7 @@ from .utils import get_analytics_columns, to_datetime_str + # For Pinterest analytics streams rate limit is 300 calls per day / per user. # once hit - response would contain `code` property with int. MAX_RATE_LIMIT_CODE = 8 diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py index 0ca0151e2db8d..271de750bcff5 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py @@ -5,11 +5,12 @@ from typing import Any, Mapping from unittest.mock import MagicMock -from airbyte_cdk.sources.streams import Stream from pytest import fixture from source_pinterest.reports import CampaignAnalyticsReport from source_pinterest.source import SourcePinterest +from airbyte_cdk.sources.streams import Stream + @fixture def test_config() -> Mapping[str, str]: diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py index 86cc6f23f8f4d..6f2f2207cdb8b 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py @@ -7,16 +7,19 @@ import pendulum import pytest import requests +from requests import Response +from source_pinterest.python_stream_auth import PinterestOauthAuthenticator + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from requests import Response -from source_pinterest.python_stream_auth import PinterestOauthAuthenticator + LOGGER = logging.getLogger(__name__) resp = Response() + class TestPinterestOauthAuthenticator: """ Test class for custom PinterestOauthAuthenticator, derived from the CDK's Oauth2Authenticator class. @@ -69,7 +72,9 @@ def test_refresh_access_token_invalid_or_expired(self, mocker, oauth): mocker.patch.object(resp, "status_code", 400) mocker.patch.object(oauth, "_wrap_refresh_token_exception", return_value=True) - with pytest.raises(AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings."): + with pytest.raises( + AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." + ): oauth.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py index 3dd38604a86eb..bfa60d227e313 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py @@ -7,11 +7,12 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from pytest import fixture from source_pinterest.streams import IncrementalPinterestSubStream +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + from .conftest import get_stream_by_name from .utils import create_requests_response @@ -135,7 +136,8 @@ def test_semi_incremental_read(requests_mock, test_config, start_date, stream_st stream.state = stream_state actual_records = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) for record in stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice) ] assert actual_records == expected_records diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index df5c903ee3475..16df2e7ad9cf4 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -24,7 +24,8 @@ ) from source_pinterest.utils import get_analytics_columns -os.environ["REQUEST_CACHE_PATH"] = '/tmp' + +os.environ["REQUEST_CACHE_PATH"] = "/tmp" def test_request_body_json(analytics_report_stream, date_range): @@ -79,18 +80,20 @@ def test_streams(test_config): def test_custom_streams(test_config): config = copy.deepcopy(test_config) - config['custom_reports'] = [{ - "name": "vadim_report", - "level": "AD_GROUP", - "granularity": "MONTH", - "click_window_days": 30, - "engagement_window_days": 30, - "view_window_days": 30, - "conversion_report_time": "TIME_OF_CONVERSION", - "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], - "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], - "start_date": "2023-01-08" - }] + config["custom_reports"] = [ + { + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08", + } + ] source = SourcePinterest() streams = source.streams(config) expected_streams_number = 33 @@ -100,18 +103,18 @@ def test_custom_streams(test_config): @pytest.mark.parametrize( ("report_name", "expected_level"), ( - [CampaignAnalyticsReport, 'CAMPAIGN'], - [CampaignTargetingReport, 'CAMPAIGN_TARGETING'], - [AdvertiserReport, 'ADVERTISER'], - [AdvertiserTargetingReport, 'ADVERTISER_TARGETING'], - [AdGroupReport, 'AD_GROUP'], - [AdGroupTargetingReport, 'AD_GROUP_TARGETING'], - [PinPromotionReport, 'PIN_PROMOTION'], - [PinPromotionTargetingReport, 'PIN_PROMOTION_TARGETING'], - [ProductGroupReport, 'PRODUCT_GROUP'], - [ProductGroupTargetingReport, 'PRODUCT_GROUP_TARGETING'], - [ProductItemReport, 'PRODUCT_ITEM'], - [KeywordReport, 'KEYWORD'] + [CampaignAnalyticsReport, "CAMPAIGN"], + [CampaignTargetingReport, "CAMPAIGN_TARGETING"], + [AdvertiserReport, "ADVERTISER"], + [AdvertiserTargetingReport, "ADVERTISER_TARGETING"], + [AdGroupReport, "AD_GROUP"], + [AdGroupTargetingReport, "AD_GROUP_TARGETING"], + [PinPromotionReport, "PIN_PROMOTION"], + [PinPromotionTargetingReport, "PIN_PROMOTION_TARGETING"], + [ProductGroupReport, "PRODUCT_GROUP"], + [ProductGroupTargetingReport, "PRODUCT_GROUP_TARGETING"], + [ProductItemReport, "PRODUCT_ITEM"], + [KeywordReport, "KEYWORD"], ), ) def test_level(test_config, report_name, expected_level): diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 6012f9d1b2118..5e615e7c2e8d3 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -5,9 +5,10 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_pinterest.source import SourcePinterest +from airbyte_cdk.utils import AirbyteTracedException + def test_check_connection(requests_mock, test_config): requests_mock.get("https://api.pinterest.com/v5/boards", status_code=200) @@ -30,8 +31,7 @@ def test_check_connection_expired_token(requests_mock, test_config): logger_mock = MagicMock() assert source.check_connection(logger_mock, test_config) == ( False, - "Unable to connect to stream boards - 401 Client Error: None " - "for url: https://api.pinterest.com/v5/oauth/token", + "Unable to connect to stream boards - 401 Client Error: None " "for url: https://api.pinterest.com/v5/oauth/token", ) diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 80f5d4aa37a75..b1ecc6bfef44b 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -8,10 +8,6 @@ import pytest import requests -from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models.airbyte_protocol import SyncMode -from airbyte_cdk.sources.declarative.types import StreamSlice -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from source_pinterest.streams import ( AnalyticsApiBackoffStrategyDecorator, NonJSONResponse, @@ -22,9 +18,15 @@ ) from source_pinterest.utils import get_analytics_columns +from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models.airbyte_protocol import SyncMode +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + from .conftest import get_stream_by_name from .utils import create_requests_response + os.environ["REQUEST_CACHE_PATH"] = "/tmp" _ANY_STREAM_NAME = "any_stream_name" _RETRY_AFTER_HEADER = "XRetry-After" @@ -80,7 +82,8 @@ def test_parse_response_with_sensitive_data(requests_mock, test_config): json={"items": [{"id": "CatalogsFeeds1", "credentials": {"password": "bla"}}]}, ) actual_response = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) ] assert actual_response == [{"id": "CatalogsFeeds1"}] @@ -122,7 +125,9 @@ def test_response_action(requests_mock, patch_base_class, http_status, expected_ ), ) @patch("time.sleep", return_value=None) -def test_declarative_stream_response_action_on_max_rate_limit_error(mock_sleep, requests_mock, test_response, status_code, expected_response_action): +def test_declarative_stream_response_action_on_max_rate_limit_error( + mock_sleep, requests_mock, test_response, status_code, expected_response_action +): response_mock = create_requests_response(requests_mock, status_code, {}) error_handler = PinterestErrorHandler(logger=MagicMock(), stream_name="any_stream_name") assert error_handler.interpret_response(response_mock).response_action == expected_response_action diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pipedrive/main.py b/airbyte-integrations/connectors/source-pipedrive/main.py index 64fe456c34fd1..4b1f33f52cccf 100644 --- a/airbyte-integrations/connectors/source-pipedrive/main.py +++ b/airbyte-integrations/connectors/source-pipedrive/main.py @@ -4,5 +4,6 @@ from source_pipedrive.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py index 961438884da60..2f34fa080e105 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping, Union import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py index 68d288dbaeef2..fd9c66cb3824c 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py @@ -7,11 +7,12 @@ from datetime import datetime from typing import List +from orjson import orjson +from source_pipedrive import SourcePipedrive + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson -from source_pipedrive import SourcePipedrive def _get_source(args: List[str]): diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py index 8070fbcfcc404..b109d02ac76ce 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-plaid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py index 3a0f562732fb4..6e0d32803f452 100644 --- a/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-plausible/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pocket/components.py b/airbyte-integrations/connectors/source-pocket/components.py index 0d5ef1cec5524..19414f6f4561e 100644 --- a/airbyte-integrations/connectors/source-pocket/components.py +++ b/airbyte-integrations/connectors/source-pocket/components.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pocket/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py index 56afb08757069..93a63daefa3c0 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py @@ -13,6 +13,7 @@ import pytz from psycopg2 import extensions, sql + catalog_write_file = "/connector/integration_tests/temp/configured_catalog_copy.json" catalog_source_file = "/connector/integration_tests/configured_catalog_template.json" catalog_incremental_write_file = "/connector/integration_tests/temp/incremental_configured_catalog_copy.json" @@ -20,12 +21,13 @@ abnormal_state_write_file = "/connector/integration_tests/temp/abnormal_state_copy.json" abnormal_state_file = "/connector/integration_tests/abnormal_state_template.json" -secret_config_file = '/connector/secrets/config.json' -secret_active_config_file = '/connector/integration_tests/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/config_cdc.json' -secret_active_config_cdc_file = '/connector/integration_tests/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/config.json" +secret_active_config_file = "/connector/integration_tests/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/config_cdc.json" +secret_active_config_cdc_file = "/connector/integration_tests/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') def connect_to_db() -> extensions.connection: with open(secret_config_file) as f: @@ -33,11 +35,7 @@ def connect_to_db() -> extensions.connection: try: conn: extensions.connection = psycopg2.connect( - dbname=secret["database"], - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + dbname=secret["database"], user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") return conn @@ -45,6 +43,7 @@ def connect_to_db() -> extensions.connection: print(f"Error connecting to the database: {error}") sys.exit(1) + def insert_records(conn: extensions.connection, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: try: cursor = conn.cursor() @@ -64,6 +63,7 @@ def insert_records(conn: extensions.connection, schema_name: str, table_name: st finally: cursor.close() + def create_schema(conn: extensions.connection, schema_name: str) -> None: try: cursor = conn.cursor() @@ -77,24 +77,25 @@ def create_schema(conn: extensions.connection, schema_name: str) -> None: finally: cursor.close() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path("/connector/integration_tests/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["schemas"] = [schema_name] - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: @@ -104,9 +105,10 @@ def write_supporting_file(schema_name: str) -> None: secret["replication_method"]["publication"] = schema_name secret["ssl_mode"] = {} secret["ssl_mode"]["mode"] = "require" - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn: extensions.connection, schema_name: str, table_name: str) -> None: try: cursor = conn.cursor() @@ -126,40 +128,37 @@ def create_table(conn: extensions.connection, schema_name: str, table_name: str) finally: cursor.close() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] + new_records = [("4", "four"), ("5", "five")] connection = connect_to_db() - table_name = 'id_and_name_cat' + table_name = "id_and_name_cat" if connection: insert_records(connection, schema_name, table_name, new_records) connection.close() + def setup(with_cdc=False): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" # Define the records to be inserted - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] # Connect to the database connection = connect_to_db() @@ -167,7 +166,7 @@ def setup(with_cdc=False): # Create the schema create_schema(connection, schema_name) create_table(connection, schema_name, table_name) - if (with_cdc): + if with_cdc: setup_cdc(connection, replication_slot_and_publication_name=schema_name) # Insert the records insert_records(connection, schema_name, table_name, records) @@ -175,6 +174,7 @@ def setup(with_cdc=False): # Close the connection connection.close() + def replication_slot_existed(connection, replication_slot_name): cursor = connection.cursor() cursor.execute("SELECT slot_name FROM pg_replication_slots;") @@ -185,22 +185,31 @@ def replication_slot_existed(connection, replication_slot_name): return True return False + def setup_cdc(connection, replication_slot_and_publication_name): cursor = connection.cursor() if replication_slot_existed(connection, replication_slot_and_publication_name): return - create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format(sql.Literal(replication_slot_and_publication_name)) + create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format( + sql.Literal(replication_slot_and_publication_name) + ) cursor.execute(create_logical_replication_query) - alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format(sql.Identifier(replication_slot_and_publication_name)) + alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format( + sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(alter_table_replica_query) - create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format(sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name)) + create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format( + sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(create_publication_query) connection.commit() + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_cdc_with_prefix(conn, prefix): try: # Connect to the PostgreSQL database @@ -224,6 +233,7 @@ def delete_cdc_with_prefix(conn, prefix): if cursor: cursor.close() + def delete_schemas_with_prefix(conn, date_prefix): try: # Connect to the PostgreSQL database @@ -252,14 +262,16 @@ def delete_schemas_with_prefix(conn, date_prefix): finally: cursor.close() + def teardown() -> None: conn = connect_to_db() today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") delete_schemas_with_prefix(conn, formatted_yesterday) delete_cdc_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: conn = connect_to_db() schema_name = load_schema_name_from_catalog() @@ -267,6 +279,7 @@ def final_teardown() -> None: delete_schemas_with_prefix(conn, schema_name) delete_cdc_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-posthog/main.py b/airbyte-integrations/connectors/source-posthog/main.py index f7e69357d9997..e6ff813e7420b 100644 --- a/airbyte-integrations/connectors/source-posthog/main.py +++ b/airbyte-integrations/connectors/source-posthog/main.py @@ -4,5 +4,6 @@ from source_posthog.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py index 268ce3822d514..a7ded5e7a7930 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py index a7c40deb21feb..7e1bb5bd1d0b0 100644 --- a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py @@ -3,11 +3,13 @@ # import pytest as pytest +from source_posthog.components import EventsCartesianProductStreamSlicer + from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption -from source_posthog.components import EventsCartesianProductStreamSlicer + stream_slicers = [ ListPartitionRouter(values=[2331], cursor_field="project_id", config={}, parameters={}), diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py index 04bbfbafe845c..5016adc4b09ff 100644 --- a/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py @@ -3,9 +3,10 @@ # -from airbyte_cdk.logger import AirbyteLogger from source_posthog import SourcePosthog +from airbyte_cdk.logger import AirbyteLogger + def test_client_wrong_credentials(): source = SourcePosthog() diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py index 51f9b12de27dd..e023a7a92b301 100644 --- a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py @@ -4,6 +4,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-prestashop/main.py b/airbyte-integrations/connectors/source-prestashop/main.py index e51c47a996aa4..1daf7c26e83b6 100644 --- a/airbyte-integrations/connectors/source-prestashop/main.py +++ b/airbyte-integrations/connectors/source-prestashop/main.py @@ -4,5 +4,6 @@ from source_prestashop.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py index 0175750fc8146..a3f42725cfcee 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py @@ -6,10 +6,11 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum +from pendulum.parsing.exceptions import ParserError + from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState -from pendulum.parsing.exceptions import ParserError @dataclass diff --git a/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-primetric/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-public-apis/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-public-apis/main.py b/airbyte-integrations/connectors/source-public-apis/main.py index c9796a4aa4ef6..d39349fbd4214 100644 --- a/airbyte-integrations/connectors/source-public-apis/main.py +++ b/airbyte-integrations/connectors/source-public-apis/main.py @@ -4,5 +4,6 @@ from source_public_apis.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py index d659c4dfe5b1a..f5b17ab6ba994 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py @@ -5,10 +5,10 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor class CustomExtractor(RecordExtractor): def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: - return [{"name": cat} for cat in response.json()["categories"]] diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py index b4927fb5a5e29..951a517434cf9 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_public_apis import SourcePublicApis +from airbyte_cdk.entrypoint import launch + def run(): source = SourcePublicApis() diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py index b9925483338da..ab25aeb7a3672 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pypi/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/main.py b/airbyte-integrations/connectors/source-python-http-tutorial/main.py index 57dce4e0679c9..e38c9bbaf28ec 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/main.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/main.py @@ -4,5 +4,6 @@ from source_python_http_tutorial.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py index 35164f2108aa4..9c354e0ab0ff4 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/setup.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/setup.py @@ -4,6 +4,7 @@ from setuptools import find_packages, setup + setup( entry_points={ "console_scripts": [ diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py index 07921116b35cd..f287a682d4989 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py +++ b/airbyte-integrations/connectors/source-python-http-tutorial/source_python_http_tutorial/source.py @@ -7,6 +7,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-qualaroo/components.py b/airbyte-integrations/connectors/source-qualaroo/components.py index 5e4e619d4d442..5c700af307b26 100644 --- a/airbyte-integrations/connectors/source-qualaroo/components.py +++ b/airbyte-integrations/connectors/source-qualaroo/components.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor @@ -15,7 +16,6 @@ class CustomAuthenticator(BasicHttpAuthenticator): @property def token(self): - key = str(self._username.eval(self.config)).encode("latin1") token = self._password.eval(self.config).encode("latin1") encoded_credentials = b64encode(b":".join((key, token))).strip() @@ -25,7 +25,6 @@ def token(self): class CustomExtractor(RecordExtractor): def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: - extracted = [] for record in response.json(): if "answered_questions" in record: diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-quickbooks/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-quickbooks/main.py b/airbyte-integrations/connectors/source-quickbooks/main.py index abeed13585f59..bce6717c1dc96 100644 --- a/airbyte-integrations/connectors/source-quickbooks/main.py +++ b/airbyte-integrations/connectors/source-quickbooks/main.py @@ -4,5 +4,6 @@ from source_quickbooks.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py index 644d8d5b227e0..77fa080207774 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-railz/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-railz/main.py b/airbyte-integrations/connectors/source-railz/main.py index bfa6b9fadb2f3..ec454b464ed94 100644 --- a/airbyte-integrations/connectors/source-railz/main.py +++ b/airbyte-integrations/connectors/source-railz/main.py @@ -4,5 +4,6 @@ from source_railz.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-railz/source_railz/components.py b/airbyte-integrations/connectors/source-railz/source_railz/components.py index 36a9351f859d3..c7c4ef88a4675 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/components.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/components.py @@ -8,6 +8,8 @@ from typing import Any, Iterable, Mapping, Optional, Union import requests +from isodate import Duration, parse_duration + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator @@ -15,7 +17,6 @@ from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator -from isodate import Duration, parse_duration @dataclass diff --git a/airbyte-integrations/connectors/source-railz/source_railz/run.py b/airbyte-integrations/connectors/source-railz/source_railz/run.py index 831b665a05da6..92d5008017b46 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/run.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_railz import SourceRailz +from airbyte_cdk.entrypoint import launch + def run(): source = SourceRailz() diff --git a/airbyte-integrations/connectors/source-railz/source_railz/source.py b/airbyte-integrations/connectors/source-railz/source_railz/source.py index d5c9010ae224a..642ebb4eb236c 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/source.py +++ b/airbyte-integrations/connectors/source-railz/source_railz/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recharge/main.py b/airbyte-integrations/connectors/source-recharge/main.py index d8ccf40b711ea..5f5e6b67eed1d 100644 --- a/airbyte-integrations/connectors/source-recharge/main.py +++ b/airbyte-integrations/connectors/source-recharge/main.py @@ -4,5 +4,6 @@ from source_recharge.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py index 80d19ac2eca28..d89d6281908c3 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/recharge_error_handler.py @@ -5,9 +5,10 @@ import logging from typing import Optional, Union +from requests import Response + from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, FailureType, ResponseAction -from requests import Response class RechargeErrorHandler(HttpStatusErrorHandler): @@ -16,7 +17,6 @@ def __init__(self, logger: logging.Logger) -> None: super().__init__(logger=logger) def interpret_response(self, response_or_exception: Optional[Union[Response, Exception]] = None) -> ErrorResolution: - if isinstance(response_or_exception, Response): content_length = int(response_or_exception.headers.get("Content-Length", 0)) incomplete_data_response = response_or_exception.status_code == 200 and content_length > len(response_or_exception.content) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index be0b9d43509d9..e2aa51028f59d 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -9,6 +9,7 @@ from airbyte_cdk.sources.streams import Stream from source_recharge.streams import Orders, RechargeTokenAuthenticator + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py index 39b44c35cf3d8..be01b454fa1cd 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py index 3581ace5712d0..a014169b9c075 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py @@ -10,6 +10,7 @@ import pendulum + START_DATE = "2023-01-01T00:00:00Z" ACCESS_TOKEN = "test_access_token" DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py index 4522eec9675e8..cb3c13212c376 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py @@ -8,6 +8,7 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + NEXT_PAGE_TOKEN = "New_Next_Page_Token" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index d6a06768d52f6..cebf1c6765490 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -47,7 +47,7 @@ def with_access_token(self, access_token: str) -> RequestBuilder: def with_old_api_version(self, api_version: str) -> RequestBuilder: self._headers["X-Recharge-Version"] = api_version return self - + def with_created_min(self, value: str) -> RequestBuilder: self._query_params["created_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) return self diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py index 91ebbbcb26e6a..b6055873bc816 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py @@ -5,12 +5,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, read_full_refresh + _STREAM_NAME = "collections" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 906fb220c3fc2..430a27d2a0ce8 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "discounts" _CURSOR_FIELD = "updated_at" @@ -31,7 +33,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py index c33a76e800c5c..4093fba40a020 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_events.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "events" _CURSOR_FIELD = "created_at" @@ -32,7 +34,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index 713678584e216..64255813f5033 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import StreamTestCase, config, get_cursor_value_from_state_message, read_full_refresh, read_incremental + _STREAM_NAME = "onetimes" _CURSOR_FIELD = "updated_at" @@ -31,7 +33,6 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - http_mocker.get( self.stream_request().with_limit(250).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py index 3970bf63fad16..0991a86f1d9bf 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_shop.py @@ -6,12 +6,14 @@ from unittest import TestCase import freezegun + from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from ..config import NOW from ..utils import StreamTestCase, read_full_refresh + _STREAM_NAME = "shop" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py index 3da0b1d06775e..475f8d95ee1d0 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -8,10 +8,11 @@ from typing import Any, List, Mapping, Optional from unittest import TestCase +from source_recharge import SourceRecharge + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode -from source_recharge import SourceRecharge from .config import ConfigBuilder from .request_builder import RequestBuilder diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index af3ba6d425b82..75f5536adf843 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -8,9 +8,10 @@ import pytest import requests -from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from source_recharge.source import Orders, RechargeTokenAuthenticator, SourceRecharge +from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction + def use_orders_deprecated_api_config( config: Mapping[str, Any] = None, @@ -88,6 +89,7 @@ def test_should_retry(self, config, http_status, headers, expected_action) -> No error_resolution = stream.get_error_handler().interpret_response(response) error_resolution.response_action == expected_action + class TestFullRefreshStreams: def generate_records(self, stream_name, count) -> Union[Mapping[str, List[Mapping[str, Any]]], Mapping[str, Any]]: if not stream_name: diff --git a/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recreation/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recruitee/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-recurly/components.py b/airbyte-integrations/connectors/source-recurly/components.py index 335c318c79444..6cebf58f980d4 100644 --- a/airbyte-integrations/connectors/source-recurly/components.py +++ b/airbyte-integrations/connectors/source-recurly/components.py @@ -3,6 +3,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py index efc25f08ce82e..78b220cebb187 100644 --- a/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-recurly/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-reply-io/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-retently/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-ringcentral/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rki-covid/main.py b/airbyte-integrations/connectors/source-rki-covid/main.py index a104106001cb1..1e4d286df619c 100644 --- a/airbyte-integrations/connectors/source-rki-covid/main.py +++ b/airbyte-integrations/connectors/source-rki-covid/main.py @@ -4,5 +4,6 @@ from source_rki_covid.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 65fe10330a2e6..ee3cc9e34ea56 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -15,7 +16,6 @@ # Basic full refresh stream class RkiCovidStream(HttpStream, ABC): - url_base = "https://api.corona-zahlen.org/" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -106,7 +106,6 @@ def path( # Basic incremental stream class IncrementalRkiCovidStream(RkiCovidStream, ABC): - state_checkpoint_interval = None @property diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py index e701d03455d4f..a4058a6f3deb0 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_incremental_streams.py @@ -3,10 +3,11 @@ # -from airbyte_cdk.models import SyncMode from pytest import fixture from source_rki_covid.source import IncrementalRkiCovidStream +from airbyte_cdk.models import SyncMode + @fixture def patch_incremental_base_class(mocker): diff --git a/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rocket-chat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-rss/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-rss/main.py b/airbyte-integrations/connectors/source-rss/main.py index 9e21c3c977934..c8e2b7f3042c9 100644 --- a/airbyte-integrations/connectors/source-rss/main.py +++ b/airbyte-integrations/connectors/source-rss/main.py @@ -4,5 +4,6 @@ from source_rss.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-rss/source_rss/components.py b/airbyte-integrations/connectors/source-rss/source_rss/components.py index 571d1a0a2e0ca..4379d281e143f 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/components.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/components.py @@ -12,12 +12,13 @@ import feedparser import pytz import requests +from dateutil.parser import parse + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.types import StreamSlice from airbyte_cdk.sources.streams.core import Stream -from dateutil.parser import parse class CustomExtractor(RecordExtractor): diff --git a/airbyte-integrations/connectors/source-rss/source_rss/run.py b/airbyte-integrations/connectors/source-rss/source_rss/run.py index 90f8a101fcfa1..c91bcd0dfb709 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/run.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_rss import SourceRss +from airbyte_cdk.entrypoint import launch + def run(): source = SourceRss() diff --git a/airbyte-integrations/connectors/source-rss/source_rss/source.py b/airbyte-integrations/connectors/source-rss/source_rss/source.py index 297b6a38c9ef8..b9e4d0882b566 100644 --- a/airbyte-integrations/connectors/source-rss/source_rss/source.py +++ b/airbyte-integrations/connectors/source-rss/source_rss/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py index 706e9eba88bea..de2ec1e2928c2 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py @@ -11,6 +11,7 @@ import pytest import yaml + pytest_plugins = ("connector_acceptance_test.plugin",) logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py index 26647e4f62ad9..9f4f69e5a0626 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py @@ -19,6 +19,9 @@ import orjson import pytest import yaml +from pydantic import BaseModel +from source_s3.v4.source import SourceS3 + from airbyte_cdk.models import ( AirbyteMessage, AirbyteStream, @@ -29,8 +32,7 @@ Type, ) from airbyte_cdk.test import entrypoint_wrapper -from pydantic import BaseModel -from source_s3.v4.source import SourceS3 + if TYPE_CHECKING: from airbyte_cdk import Source @@ -70,6 +72,7 @@ def expect_exception(self) -> bool: def instance_name(self) -> str: return self.config_path.stem + def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: all_tests_config = yaml.safe_load(ACCEPTANCE_TEST_CONFIG_PATH.read_text()) return [ @@ -78,6 +81,7 @@ def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: if "iam_role" not in test["config_path"] ] + # TODO: Convert to a CDK class for better reuse and portability. # class TestSourceAcceptanceTestSuiteBase: # """Test suite for acceptance tests.""" diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py index 6f38722d30cc7..2596833e33a41 100644 --- a/airbyte-integrations/connectors/source-s3/main.py +++ b/airbyte-integrations/connectors/source-s3/main.py @@ -4,5 +4,6 @@ from source_s3.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py b/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py index 88908adc107d7..1351622538395 100644 --- a/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py +++ b/airbyte-integrations/connectors/source-s3/scripts/fetch_test_secrets.py @@ -28,6 +28,7 @@ import airbyte as ab from airbyte.secrets import GoogleGSMSecretManager, SecretHandle + AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing" CONNECTOR_NAME = "source-s3" MISSING_ONLY = True diff --git a/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py b/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py index f9b8f6a85523e..969c27f609dc5 100644 --- a/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py +++ b/airbyte-integrations/connectors/source-s3/scripts/rotate_creds.py @@ -30,6 +30,7 @@ from airbyte.secrets import get_secret from airbyte.secrets.google_gsm import GoogleGSMSecretManager, GSMSecretHandle + AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing" CONNECTOR_NAME = "source-s3" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py index 69799dfa2daeb..4ad361ac89aa5 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py @@ -7,9 +7,11 @@ from traceback import format_exc from typing import Any, List, Mapping, Optional, Tuple -from airbyte_cdk import AbstractSource, ConnectorSpecification, DestinationSyncMode, Stream, SyncMode from wcmatch.glob import GLOBSTAR, SPLIT, globmatch +from airbyte_cdk import AbstractSource, ConnectorSpecification, DestinationSyncMode, Stream, SyncMode + + # ideas on extending this to handle multiple streams: # - "dataset" is currently the name of the single table/stream. We could allow comma-split table names in this string for many streams. # - "path_pattern" currently uses https://facelessuser.github.io/wcmatch/glob/ to match a single string pattern (can be multiple | separated) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py index a623f8ccf4e44..6e206b720a02e 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/spec.py @@ -15,6 +15,7 @@ from .formats.jsonl_spec import JsonlFormat from .formats.parquet_spec import ParquetFormat + # To implement your provider specific spec, inherit from SourceFilesAbstractSpec and add provider-specific settings e.g.: # class SourceS3Spec(SourceFilesAbstractSpec, BaseModel): diff --git a/airbyte-integrations/connectors/source-s3/source_s3/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/stream.py index b99632dbdc51c..6ea5ed89e9694 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/stream.py @@ -6,11 +6,12 @@ from typing import Any, Iterator, Mapping import pendulum -from airbyte_cdk import AirbyteTracedException, FailureType from boto3 import session as boto3session from botocore import UNSIGNED from botocore.config import Config from botocore.exceptions import ClientError + +from airbyte_cdk import AirbyteTracedException, FailureType from source_s3.s3_utils import make_s3_client from .s3file import S3File @@ -51,9 +52,7 @@ def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[ # list_objects_v2 doesn't like a None value for ContinuationToken # so we don't set it if we don't have one. if ctoken: - kwargs = dict( - Bucket=provider["bucket"], Prefix=provider.get("path_prefix", ""), ContinuationToken=ctoken - ) # type: ignore[unreachable] + kwargs = dict(Bucket=provider["bucket"], Prefix=provider.get("path_prefix", ""), ContinuationToken=ctoken) # type: ignore[unreachable] else: kwargs = dict(Bucket=provider["bucket"], Prefix=provider.get("path_prefix", "")) try: diff --git a/airbyte-integrations/connectors/source-s3/source_s3/utils.py b/airbyte-integrations/connectors/source-s3/source_s3/utils.py index 9118ec151ff8f..f3715bb367a72 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/utils.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/utils.py @@ -11,6 +11,7 @@ import dill import orjson + from airbyte_cdk.models import AirbyteMessage, AirbyteMessageSerializer diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index 5af899aea95ca..cdc66ab06fc70 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -5,11 +5,12 @@ from typing import Any, Dict, Optional import dpath.util -from airbyte_cdk import is_cloud_environment -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords from pydantic.v1 import AnyUrl, Field, root_validator from pydantic.v1.error_wrappers import ValidationError +from airbyte_cdk import is_cloud_environment +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords + class Config(AbstractFileBasedSpec): """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py index 0e6cf7528eee6..ab60ecf088a1c 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor from airbyte_cdk.sources.file_based.types import StreamState + logger = logging.Logger("source-S3") diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py index 4d04411a66942..c6db739caf2f2 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py @@ -12,6 +12,7 @@ from source_s3.source_files_abstract.formats.jsonl_spec import JsonlFormat from source_s3.source_files_abstract.formats.parquet_spec import ParquetFormat + SECONDS_FORMAT = "%Y-%m-%dT%H:%M:%SZ" MICROS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py index 9991c444ff640..2e3ac647a3a01 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -10,6 +10,7 @@ from typing import Any, Dict, Mapping, Optional import orjson + from airbyte_cdk import ( AirbyteEntrypoint, ConnectorSpecification, @@ -34,6 +35,7 @@ from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer from source_s3.v4.stream_reader import SourceS3StreamReader + _V3_DEPRECATION_FIELD_MAPPING = { "dataset": "streams.name", "format": "streams.format", diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py index aff8c257686d0..9c6051c8dd16f 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -14,18 +14,20 @@ import psutil import pytz import smart_open -from airbyte_cdk import FailureType -from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError, FileSizeLimitError -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient from botocore.client import Config as ClientConfig from botocore.credentials import RefreshableCredentials from botocore.exceptions import ClientError from botocore.session import get_session +from typing_extensions import override + +from airbyte_cdk import FailureType +from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError, FileSizeLimitError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_s3.v4.config import Config from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler -from typing_extensions import override + AWS_EXTERNAL_ID = getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 4f475b80c7975..12d0ed1ff8f0c 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -5,10 +5,12 @@ import zipfile from typing import IO, List, Optional, Tuple, Union -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient + +from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_s3.v4.config import Config + # Buffer constants BUFFER_SIZE_DEFAULT = 1024 * 1024 MAX_BUFFER_SIZE_DEFAULT: int = 16 * BUFFER_SIZE_DEFAULT diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py index 13e84edd6cb6e..1f2e8c726162f 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py @@ -9,6 +9,7 @@ from pydantic.v1.error_wrappers import ValidationError from source_s3.v4.config import Config + logger = logging.Logger("") diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py index f06e8fd9ae7c8..8b064c65df266 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py @@ -7,11 +7,12 @@ from unittest.mock import Mock import pytest +from source_s3.v4.cursor import Cursor + from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor -from source_s3.v4.cursor import Cursor def _create_datetime(dt: str) -> datetime: diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py index cf4a82f8d3d0e..04a23079910b5 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py @@ -8,6 +8,7 @@ from source_s3.v4 import Config, SourceS3, SourceS3StreamReader + _V3_FIELDS = ["dataset", "format", "path_pattern", "provider", "schema"] TEST_FILES_FOLDER = Path(__file__).resolve().parent.parent.joinpath("sample_files") diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 02e76e5790ea6..2512d5672daaa 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -11,16 +11,18 @@ from unittest.mock import ANY, MagicMock, Mock, patch import pytest -from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec -from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError -from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode -from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.stub import Stubber from moto import mock_sts from pydantic.v1 import AnyUrl from source_s3.v4.config import Config from source_s3.v4.stream_reader import SourceS3StreamReader +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile + + logger = logging.Logger("") endpoint_values = ["https://fake.com", None] @@ -124,7 +126,7 @@ def test_get_matching_files( except Exception as exc: raise exc - with patch.object(SourceS3StreamReader, 's3_client', new_callable=MagicMock) as mock_s3_client: + with patch.object(SourceS3StreamReader, "s3_client", new_callable=MagicMock) as mock_s3_client: _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages) files = list(reader.get_matching_files(globs, None, logger)) assert set(f.uri for f in files) == expected_uris @@ -134,27 +136,33 @@ def _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages): responses = [] if multiple_pages and len(mocked_response) > 1: # Split the mocked_response for pagination simulation - first_half = mocked_response[:len(mocked_response) // 2] - second_half = mocked_response[len(mocked_response) // 2:] - - responses.append({ - "IsTruncated": True, - "Contents": first_half, - "KeyCount": len(first_half), - "NextContinuationToken": "token", - }) - - responses.append({ - "IsTruncated": False, - "Contents": second_half, - "KeyCount": len(second_half), - }) + first_half = mocked_response[: len(mocked_response) // 2] + second_half = mocked_response[len(mocked_response) // 2 :] + + responses.append( + { + "IsTruncated": True, + "Contents": first_half, + "KeyCount": len(first_half), + "NextContinuationToken": "token", + } + ) + + responses.append( + { + "IsTruncated": False, + "Contents": second_half, + "KeyCount": len(second_half), + } + ) else: - responses.append({ - "IsTruncated": False, - "Contents": mocked_response, - "KeyCount": len(mocked_response), - }) + responses.append( + { + "IsTruncated": False, + "Contents": mocked_response, + "KeyCount": len(mocked_response), + } + ) def list_objects_v2_side_effect(Bucket, Prefix=None, ContinuationToken=None, **kwargs): if ContinuationToken == "token": @@ -252,7 +260,13 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): mock_s3_client_instance.download_file.return_value = None reader = SourceS3StreamReader() - reader.config = Config(bucket="test", aws_access_key_id="test", aws_secret_access_key="test", streams=[], delivery_method= { "delivery_type": "use_file_transfer" }) + reader.config = Config( + bucket="test", + aws_access_key_id="test", + aws_secret_access_key="test", + streams=[], + delivery_method={"delivery_type": "use_file_transfer"}, + ) try: reader.config = Config( bucket="test", @@ -260,14 +274,14 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): aws_secret_access_key="test", streams=[], endpoint=None, - delivery_method={"delivery_type": "use_file_transfer"} + delivery_method={"delivery_type": "use_file_transfer"}, ) except Exception as exc: raise exc test_file_path = "directory/file.txt" result = reader.get_file(RemoteFile(uri="", last_modified=datetime.now()), test_file_path, logger) - assert result == {'bytes': 100, 'file_relative_path': ANY, 'file_url': ANY} + assert result == {"bytes": 100, "file_relative_path": ANY, "file_url": ANY} assert result["file_url"].endswith(test_file_path) @@ -317,27 +331,20 @@ def test_get_iam_s3_client(boto3_client_mock): # Assertions to validate the s3 client assert s3_client is not None + @pytest.mark.parametrize( "start_date, last_modified_date, expected_result", ( # True when file is new or modified after given start_date - ( - datetime.now() - timedelta(days=180), - datetime.now(), - True - ), + (datetime.now() - timedelta(days=180), datetime.now(), True), ( datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), - True + True, ), # False when file is older than given start_date - ( - datetime.now(), - datetime.now() - timedelta(days=180), - False - ) - ) + (datetime.now(), datetime.now() - timedelta(days=180), False), + ), ) def test_filter_file_by_start_date(start_date: datetime, last_modified_date: datetime, expected_result: bool) -> None: reader = SourceS3StreamReader() @@ -347,7 +354,7 @@ def test_filter_file_by_start_date(start_date: datetime, last_modified_date: dat aws_access_key_id="test", aws_secret_access_key="test", streams=[], - start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ") + start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), ) assert expected_result == reader.is_modified_after_start_date(last_modified_date) diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py index 829350b465603..7372ebf129c4c 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py @@ -10,10 +10,12 @@ import pytest import requests_mock +from source_salesforce.source import SourceSalesforce + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream from airbyte_cdk.test.catalog_builder import CatalogBuilder -from source_salesforce.source import SourceSalesforce + HERE = Path(__file__).parent _ANY_CATALOG = CatalogBuilder().build() diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py index 1742ef72a9236..50926148b6313 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py @@ -12,11 +12,13 @@ import pendulum import pytest import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder + + HERE = Path(__file__).parent NOTE_CONTENT = "It's the note for integration test" @@ -51,7 +53,11 @@ def stream_name(): @pytest.fixture(scope="module") def stream(input_sandbox_config, stream_name, sf): - return SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE).generate_streams(input_sandbox_config, {stream_name: None}, sf)[0]._legacy_stream + return ( + SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) + .generate_streams(input_sandbox_config, {stream_name: None}, sf)[0] + ._legacy_stream + ) def _encode_content(text): diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py index d03de9ed39e8f..d6820c89bfbe0 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py @@ -2612,11 +2612,12 @@ import json + result = [] for stream in json.loads(x): stream["stream_descriptor"] = stream.pop("streamDescriptor") stream["stream_state"] = stream.pop("streamState") - y = { + y = { "type": "STREAM", "stream": stream, } diff --git a/airbyte-integrations/connectors/source-salesforce/main.py b/airbyte-integrations/connectors/source-salesforce/main.py index 67536217f497f..5ae62f6673353 100644 --- a/airbyte-integrations/connectors/source-salesforce/main.py +++ b/airbyte-integrations/connectors/source-salesforce/main.py @@ -5,5 +5,6 @@ from source_salesforce.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index 688962a4f8dd4..1317006b402ba 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -7,16 +7,18 @@ from typing import Any, List, Mapping, Optional, Tuple import requests # type: ignore[import] +from requests import adapters as request_adapters +from requests.exceptions import RequestException # type: ignore[import] + from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, StreamDescriptor from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.utils import AirbyteTracedException -from requests import adapters as request_adapters -from requests.exceptions import RequestException # type: ignore[import] from .exceptions import TypeSalesforceException from .rate_limiting import SalesforceErrorHandler, default_backoff_handler from .utils import filter_streams_by_criteria + STRING_TYPES = [ "byte", "combobox", diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py index a4fcca0c1e7a3..179c45ea65f42 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/availability_strategy.py @@ -6,9 +6,11 @@ import typing from typing import Optional, Tuple +from requests import HTTPError, codes + from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from requests import HTTPError, codes + if typing.TYPE_CHECKING: from airbyte_cdk.sources import Source diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py index 2fa5db0588052..13948c7131dee 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py @@ -9,10 +9,12 @@ import backoff import requests +from requests import codes, exceptions # type: ignore[import] + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from requests import codes, exceptions # type: ignore[import] + RESPONSE_CONSUMPTION_EXCEPTIONS = ( # We've had a couple of customers with ProtocolErrors, namely: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index a4236f1ff7284..ac9a1af9d0ddb 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -8,6 +8,10 @@ import isodate import pendulum +from dateutil.relativedelta import relativedelta +from pendulum.parsing.exceptions import ParserError +from requests import codes, exceptions # type: ignore[import] + from airbyte_cdk.logger import AirbyteLogFormatter from airbyte_cdk.models import ( AirbyteMessage, @@ -30,9 +34,6 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.utils.schema_helpers import InternalConfig from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from dateutil.relativedelta import relativedelta -from pendulum.parsing.exceptions import ParserError -from requests import codes, exceptions # type: ignore[import] from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .streams import ( @@ -46,6 +47,7 @@ RestSalesforceSubStream, ) + _DEFAULT_CONCURRENCY = 10 _MAX_CONCURRENCY = 10 logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 6f0ca9a0f60e8..2de1867674cb6 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -11,6 +11,9 @@ import pendulum import requests # type: ignore[import] +from pendulum import DateTime # type: ignore[attr-defined] +from requests import exceptions + from airbyte_cdk import ( BearerAuthenticator, CursorPaginationStrategy, @@ -46,13 +49,12 @@ from airbyte_cdk.sources.streams.http import HttpClient, HttpStream, HttpSubStream from airbyte_cdk.sources.types import StreamState from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum import DateTime # type: ignore[attr-defined] -from requests import exceptions from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .availability_strategy import SalesforceAvailabilityStrategy from .rate_limiting import BulkNotSupportedException, SalesforceErrorHandler, default_backoff_handler + # https://stackoverflow.com/a/54517228 CSV_FIELD_SIZE_LIMIT = int(ctypes.c_ulong(-1).value // 2) csv.field_size_limit(CSV_FIELD_SIZE_LIMIT) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 1a59386a1b952..a571cada4f53a 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -13,6 +13,20 @@ import freezegun import pytest import requests_mock +from config_builder import ConfigBuilder +from conftest import generate_stream +from salesforce_job_response_builder import JobInfoResponseBuilder +from source_salesforce.api import Salesforce +from source_salesforce.source import SourceSalesforce +from source_salesforce.streams import ( + CSV_FIELD_SIZE_LIMIT, + BulkIncrementalSalesforceStream, + BulkSalesforceStream, + BulkSalesforceSubStream, + IncrementalRestSalesforceStream, + RestSalesforceStream, +) + from airbyte_cdk.models import ( AirbyteStateBlob, AirbyteStream, @@ -28,19 +42,7 @@ from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils import AirbyteTracedException -from config_builder import ConfigBuilder -from conftest import generate_stream -from salesforce_job_response_builder import JobInfoResponseBuilder -from source_salesforce.api import Salesforce -from source_salesforce.source import SourceSalesforce -from source_salesforce.streams import ( - CSV_FIELD_SIZE_LIMIT, - BulkIncrementalSalesforceStream, - BulkSalesforceStream, - BulkSalesforceSubStream, - IncrementalRestSalesforceStream, - RestSalesforceStream, -) + _A_CHUNKED_RESPONSE = [b"first chunk", b"second chunk"] _A_JSON_RESPONSE = {"id": "any id"} @@ -100,9 +102,7 @@ def test_stream_slice_step_validation(stream_slice_step: str, expected_error_mes ), ], ) -def test_login_authentication_error_handler( - stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg -): +def test_login_authentication_error_handler(stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg): source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) logger = logging.getLogger("airbyte") requests_mock.register_uri( @@ -557,13 +557,21 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True) job_id_1 = "fake_job_1" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}", [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_1}", + [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_1}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}/results", text="Field1,LastModifiedDate,ID\ntest,2023-01-15,1") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_1}") job_id_2 = "fake_job_2" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_2}", [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_2}", + [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_2}") requests_mock.register_uri( "GET", _bulk_stream_path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22" @@ -574,7 +582,11 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api queries_history = requests_mock.register_uri( "POST", _bulk_stream_path(), [{"json": {"id": job_id_1}}, {"json": {"id": job_id_2}}, {"json": {"id": job_id_3}}] ) - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}", [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_3}", + [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_3}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_3}") @@ -586,18 +598,24 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api # assert request params: has requests might not be performed in a specific order because of concurrent CDK, we match on any request all_requests = {request.text for request in queries_history.request_history} - assert any([ - "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" - in request for request in all_requests - ]) + assert any( + [ + "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" in request + for request in all_requests + ] + ) # as the execution is concurrent, we can only assert the last state message here last_actual_state = [item.state.stream.stream_state for item in result if item.type == Type.STATE][-1] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index 92df1b20876e9..348d2de1b0816 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -7,13 +7,15 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalogSerializer -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.state_builder import StateBuilder from config_builder import ConfigBuilder from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalogSerializer +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.state_builder import StateBuilder + + _ANY_CATALOG = CatalogBuilder().build() _ANY_CONFIG = ConfigBuilder().build() _ANY_STATE = StateBuilder().build() diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py index a5156457d1f25..1aa5376552cbb 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py @@ -8,15 +8,17 @@ from unittest import TestCase import freezegun +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from salesforce_job_response_builder import JobCreateResponseBuilder, JobInfoResponseBuilder +from source_salesforce.streams import BulkSalesforceStream + from airbyte_cdk.models import AirbyteStreamStatus, SyncMode from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse -from config_builder import ConfigBuilder from integration.test_rest_stream import create_http_request as create_standard_http_request from integration.test_rest_stream import create_http_response as create_standard_http_response from integration.utils import create_base_url, given_authentication, given_stream, read -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from salesforce_job_response_builder import JobCreateResponseBuilder, JobInfoResponseBuilder -from source_salesforce.streams import BulkSalesforceStream + _A_FIELD_NAME = "a_field" _ANOTHER_FIELD_NAME = "another_field" @@ -25,7 +27,9 @@ _CLIENT_SECRET = "a_client_secret" _CURSOR_FIELD = "SystemModstamp" _INCREMENTAL_FIELDS = [_A_FIELD_NAME, _CURSOR_FIELD] -_INCREMENTAL_SCHEMA_BUILDER = SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") # re-using same fields as _INCREMENTAL_FIELDS +_INCREMENTAL_SCHEMA_BUILDER = ( + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") +) # re-using same fields as _INCREMENTAL_FIELDS _INSTANCE_URL = "https://instance.salesforce.com" _JOB_ID = "a-job-id" _ANOTHER_JOB_ID = "another-job-id" @@ -60,17 +64,16 @@ def _calculate_start_time(start_time: datetime) -> datetime: def _build_job_creation_request(query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps({"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"}), + ) def _make_sliced_job_request(lower_boundary: datetime, upper_boundary: datetime, fields: List[str]) -> HttpRequest: - return _build_job_creation_request(f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + return _build_job_creation_request( + f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(fields: List[str]) -> HttpRequest: @@ -78,7 +81,6 @@ def _make_full_job_request(fields: List[str]) -> HttpRequest: class BulkStreamTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN) @@ -168,7 +170,9 @@ def test_given_type_when_read_then_field_is_casted_with_right_type(self) -> None @freezegun.freeze_time(_NOW.isoformat()) def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -192,7 +196,9 @@ def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: @freezegun.freeze_time(_NOW.isoformat()) def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -220,7 +226,9 @@ def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) @freezegun.freeze_time(_NOW.isoformat()) def test_given_specific_encoding_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -338,7 +346,17 @@ def test_given_non_transient_error_on_job_creation_when_read_then_fail_sync(self given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME]), - HttpResponse(json.dumps([{"errorCode": "API_ERROR", "message": "Implementation restriction... "}]), 400), + HttpResponse( + json.dumps( + [ + { + "errorCode": "API_ERROR", + "message": "Implementation restriction... ", + } + ] + ), + 400, + ), ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -526,11 +544,21 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) self._config.start_date(start_date).stream_slice_step("P7D") given_stream(self._http_mocker, _BASE_URL, _STREAM_WITH_PARENT_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) - self._create_sliced_job_with_records(start_date, first_upper_boundary, _PARENT_STREAM_NAME, "first_parent_slice_job_id", [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}]) - self._create_sliced_job_with_records(first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}]) + self._create_sliced_job_with_records( + start_date, + first_upper_boundary, + _PARENT_STREAM_NAME, + "first_parent_slice_job_id", + [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}], + ) + self._create_sliced_job_with_records( + first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}] + ) self._http_mocker.post( - self._build_job_creation_request(f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')"), + self._build_job_creation_request( + f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')" + ), JobCreateResponseBuilder().with_id(_JOB_ID).build(), ) self._http_mocker.get( @@ -547,10 +575,16 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) assert len(output.records) == 1 - def _create_sliced_job(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int) -> None: - self._create_sliced_job_with_records(lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count)) + def _create_sliced_job( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int + ) -> None: + self._create_sliced_job_with_records( + lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count) + ) - def _create_sliced_job_with_records(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]]) -> None: + def _create_sliced_job_with_records( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]] + ) -> None: self._http_mocker.post( self._make_sliced_job_request(lower_boundary, upper_boundary, stream_name, list(records[0].keys())), JobCreateResponseBuilder().with_id(job_id).build(), @@ -581,8 +615,12 @@ def _create_csv(self, headers: List[str], data: List[Dict[str, str]], dialect: s writer.writerow(line) return csvfile.getvalue() - def _make_sliced_job_request(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str]) -> HttpRequest: - return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + def _make_sliced_job_request( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str] + ) -> HttpRequest: + return self._build_job_creation_request( + f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(self, fields: List[str], stream_name: str = _STREAM_NAME) -> HttpRequest: return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name}") @@ -601,14 +639,13 @@ def _generate_csv(self, records: List[Dict[str, str]]) -> str: for record in records: csv_entry.append(",".join([record[key] for key in keys])) - entries = '\n'.join(csv_entry) + entries = "\n".join(csv_entry) return f"{','.join(keys)}\n{entries}" def _build_job_creation_request(self, query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps( + {"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"} + ), + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py index 85805d29cb749..d61328f9c5ba7 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py @@ -7,15 +7,17 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import AirbyteStateBlob, SyncMode -from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse -from airbyte_cdk.test.state_builder import StateBuilder from config_builder import ConfigBuilder -from integration.utils import create_base_url, given_authentication, given_stream, read from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder from source_salesforce.api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS from source_salesforce.streams import LOOKBACK_SECONDS +from airbyte_cdk.models import AirbyteStateBlob, SyncMode +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.state_builder import StateBuilder +from integration.utils import create_base_url, given_authentication, given_stream, read + + _A_FIELD_NAME = "a_field" _CLIENT_ID = "a_client_id" _CLIENT_SECRET = "a_client_secret" @@ -31,7 +33,7 @@ def create_http_request(stream_name: str, field_names: List[str], access_token: Optional[str] = None) -> HttpRequest: return HttpRequest( f"{_BASE_URL}/queryAll?q=SELECT+{','.join(field_names)}+FROM+{stream_name}+", - headers={"Authorization": f"Bearer {access_token}"} if access_token else None + headers={"Authorization": f"Bearer {access_token}"} if access_token else None, ) @@ -40,7 +42,10 @@ def create_http_response(field_names: List[str], record_count: int = 1) -> HttpR This method does not handle field types for now which may cause some test failures on change if we start considering using some fields for calculation. One example of that would be cursor field parsing to datetime. """ - records = [{field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} for i in range(record_count)] + records = [ + {field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} + for i in range(record_count) + ] return HttpResponse(json.dumps({"records": records})) @@ -64,7 +69,6 @@ def _calculate_start_time(start_time: datetime) -> datetime: @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN) @@ -77,7 +81,7 @@ def test_given_error_on_fetch_chunk_of_properties_when_read_then_retry(self, htt [ HttpResponse("", status_code=406), create_http_response([_A_FIELD_NAME], record_count=1), - ] + ], ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -94,7 +98,12 @@ def setUp(self) -> None: self._http_mocker.__enter__() given_authentication(self._http_mocker, _CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN, _INSTANCE_URL) - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime")) + given_stream( + self._http_mocker, + _BASE_URL, + _STREAM_NAME, + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime"), + ) def tearDown(self) -> None: self._http_mocker.__exit__(None, None, None) @@ -102,11 +111,13 @@ def tearDown(self) -> None: def test_given_no_state_when_read_then_start_sync_from_start(self) -> None: start = _calculate_start_time(_NOW - timedelta(days=5)) # as the start comes from the config, we can't use the same format as `_to_url` - start_format_url = urllib.parse.quote_plus(start.strftime('%Y-%m-%dT%H:%M:%SZ')) + start_format_url = urllib.parse.quote_plus(start.strftime("%Y-%m-%dT%H:%M:%SZ")) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME], record_count=1), ) @@ -119,13 +130,22 @@ def test_given_sequential_state_when_read_then_migrate_to_partitioned_state(self start = _calculate_start_time(_NOW - timedelta(days=10)) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) - output = read(_STREAM_NAME, SyncMode.incremental, self._config, StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")})) + output = read( + _STREAM_NAME, + SyncMode.incremental, + self._config, + StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")}), + ) - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_update_state(self) -> None: missing_chunk = (_NOW - timedelta(days=5), _NOW - timedelta(days=3)) @@ -138,21 +158,27 @@ def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_upda "slices": [ {"start": start.strftime("%Y-%m-%dT%H:%M:%S.000") + "Z", "end": _to_partitioned_datetime(missing_chunk[0])}, {"start": _to_partitioned_datetime(missing_chunk[1]), "end": _to_partitioned_datetime(most_recent_state_value)}, - ] - } + ], + }, ) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) output = read(_STREAM_NAME, SyncMode.incremental, self._config, state) # the start is granular to the second hence why we have `000` in terms of milliseconds - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py index 3fa57dfbd0a1d..cd380be8ee2c0 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py @@ -4,15 +4,17 @@ from unittest import TestCase import pytest +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from source_salesforce import SourceSalesforce + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from config_builder import ConfigBuilder from integration.utils import create_base_url, given_authentication, given_stream -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from source_salesforce import SourceSalesforce + _CLIENT_ID = "a_client_id" _CLIENT_SECRET = "a_client_secret" @@ -25,13 +27,10 @@ class StreamGenerationTest(TestCase): - def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN).build() self._source = SourceSalesforce( - CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), - self._config, - StateBuilder().build() + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), self._config, StateBuilder().build() ) self._http_mocker = HttpMocker() @@ -48,10 +47,7 @@ def test_given_transient_error_fetching_schema_when_streams_then_retry(self) -> ) self._http_mocker.get( HttpRequest(f"{_BASE_URL}/sobjects/{_STREAM_NAME}/describe"), - [ - HttpResponse("", status_code=406), - SalesforceDescribeResponseBuilder().field("a_field_name").build() - ] + [HttpResponse("", status_code=406), SalesforceDescribeResponseBuilder().field("a_field_name").build()], ) streams = self._source.streams(self._config) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py index b547a297c439e..bae6d4bd14452 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py @@ -3,6 +3,10 @@ import json from typing import Any, Dict, Optional +from config_builder import ConfigBuilder +from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder +from source_salesforce import SourceSalesforce + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -11,9 +15,7 @@ from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder -from source_salesforce import SourceSalesforce + _API_VERSION = "v57.0" @@ -35,7 +37,7 @@ def read( sync_mode: SyncMode, config_builder: Optional[ConfigBuilder] = None, state_builder: Optional[StateBuilder] = None, - expecting_exception: bool = False + expecting_exception: bool = False, ) -> EntrypointOutput: catalog = _catalog(stream_name, sync_mode) config = config_builder.build() if config_builder else ConfigBuilder().build() @@ -43,12 +45,19 @@ def read( return entrypoint_read(_source(catalog, config, state), config, catalog, state, expecting_exception) -def given_authentication(http_mocker: HttpMocker, client_id: str, client_secret: str, refresh_token: str, instance_url: str, access_token: str = "any_access_token") -> None: +def given_authentication( + http_mocker: HttpMocker, + client_id: str, + client_secret: str, + refresh_token: str, + instance_url: str, + access_token: str = "any_access_token", +) -> None: http_mocker.post( HttpRequest( "https://login.salesforce.com/services/oauth2/token", query_params=ANY_QUERY_PARAMS, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}", ), HttpResponse(json.dumps({"access_token": access_token, "instance_url": instance_url})), ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py index 55bc8b8f65ddf..76af9b7eaf382 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py @@ -9,18 +9,18 @@ class JobCreateResponseBuilder: def __init__(self) -> None: self._response = { - "id": "any_id", - "operation": "query", - "object": "Account", - "createdById": "005R0000000GiwjIAC", - "createdDate": "2018-12-17T21:00:17.000+0000", - "systemModstamp": "2018-12-17T21:00:17.000+0000", - "state": "UploadComplete", - "concurrencyMode": "Parallel", - "contentType": "CSV", - "apiVersion": 46.0, - "lineEnding": "LF", - "columnDelimiter": "COMMA" + "id": "any_id", + "operation": "query", + "object": "Account", + "createdById": "005R0000000GiwjIAC", + "createdDate": "2018-12-17T21:00:17.000+0000", + "systemModstamp": "2018-12-17T21:00:17.000+0000", + "state": "UploadComplete", + "concurrencyMode": "Parallel", + "contentType": "CSV", + "apiVersion": 46.0, + "lineEnding": "LF", + "columnDelimiter": "COMMA", } self._status_code = 200 @@ -52,11 +52,11 @@ def with_state(self, state: str) -> "JobInfoResponseBuilder": def with_status_code(self, status_code: int) -> "JobInfoResponseBuilder": self._status_code = status_code return self - + def with_error_message(self, error_message: str) -> "JobInfoResponseBuilder": self._response["errorMessage"] = error_message return self - + def get_response(self) -> any: return self._response diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py index 2c8abd44f859d..cd00f723774ab 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_availability_strategy.py @@ -4,10 +4,12 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.streams import Stream from requests import HTTPError, Response from source_salesforce.availability_strategy import SalesforceAvailabilityStrategy +from airbyte_cdk.sources.streams import Stream + + _NO_SOURCE = None diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py index c6b9d1ea17317..c00beb8981c1f 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py @@ -6,11 +6,13 @@ import pytest import requests import requests_mock -from airbyte_cdk.models import FailureType -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from requests.exceptions import ChunkedEncodingError from source_salesforce.rate_limiting import BulkNotSupportedException, SalesforceErrorHandler +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + + _ANY = "any" _ANY_BASE_URL = "https://any-base-url.com" _SF_API_VERSION = "v57.0" @@ -20,18 +22,32 @@ class SalesforceErrorHandlerTest(TestCase): def setUp(self) -> None: self._error_handler = SalesforceErrorHandler() - def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}]) + def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported( + self, + ) -> None: + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_compound_data_error_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_request_limit_exceeded_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) @@ -41,12 +57,24 @@ def test_given_limit_exceeded_on_job_creation_when_interpret_response_then_raise self._error_handler.interpret_response(response) def test_given_query_not_supported_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}]) + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_txn_security_metering_error_when_interpret_response_then_raise_config_error(self) -> None: - response = self._create_response("GET", self._url_for_job_creation() + "/job_id", 400, [{"errorCode": "TXN_SECURITY_METERING_ERROR", "message": "We can't complete the action because enabled transaction security policies took too long to complete."}]) + response = self._create_response( + "GET", + self._url_for_job_creation() + "/job_id", + 400, + [ + { + "errorCode": "TXN_SECURITY_METERING_ERROR", + "message": "We can't complete the action because enabled transaction security policies took too long to complete.", + } + ], + ) error_resolution = self._error_handler.interpret_response(response) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py index 25ea70f6362cf..b2f335f594e80 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py @@ -4,20 +4,24 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import SyncMode from config_builder import ConfigBuilder from conftest import generate_stream, mock_stream_api from source_salesforce.api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS +from airbyte_cdk.models import SyncMode + + _NOW = datetime.fromisoformat("2020-01-01T00:00:00+00:00") _STREAM_NAME = UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] + @freezegun.freeze_time(time_to_freeze=_NOW) class IncrementalSliceGenerationTest(TestCase): """ For this, we will be testing with UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] as bulk stream slicing actually creates jobs. We will assume the bulk one usese the same logic. """ + def test_given_start_within_slice_range_when_stream_slices_then_return_one_slice_considering_10_minutes_lookback(self) -> None: config = ConfigBuilder().start_date(_NOW - timedelta(days=15)).stream_slice_step("P30D").build() stream = generate_stream(_STREAM_NAME, config, mock_stream_api(config)) @@ -34,5 +38,5 @@ def test_given_slice_range_smaller_than_now_minus_start_date_when_stream_slices_ assert slices == [ {"start_date": "2019-11-22T00:00:00.000+00:00", "end_date": "2019-12-22T00:00:00.000+00:00"}, - {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"} + {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"}, ] diff --git a/airbyte-integrations/connectors/source-salesloft/components.py b/airbyte-integrations/connectors/source-salesloft/components.py index 5475aa13ca33f..4d94758110e52 100644 --- a/airbyte-integrations/connectors/source-salesloft/components.py +++ b/airbyte-integrations/connectors/source-salesloft/components.py @@ -10,7 +10,6 @@ @dataclass class SingleUseOauth2Authenticator(DeclarativeSingleUseRefreshTokenOauth2Authenticator): - config: Config def __post_init__(self): diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-secoda/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-senseforce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sentry/main.py b/airbyte-integrations/connectors/source-sentry/main.py index 1c7adc746e974..c5f41c2a45028 100644 --- a/airbyte-integrations/connectors/source-sentry/main.py +++ b/airbyte-integrations/connectors/source-sentry/main.py @@ -4,5 +4,6 @@ from source_sentry.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py index 40282ee1ff624..44ad79700b2f3 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/run.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/run.py @@ -8,9 +8,10 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_sentry import SourceSentry diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py index 0c5af692c5af2..ab243cf755d66 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "auth_token": "test token", "organization": "test organization", "project": "test project", - "hostname": "sentry.io" + "hostname": "sentry.io", } def build(self) -> Dict[str, Any]: diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py index 85987dd3d2dde..3c29f713d5dd0 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py @@ -3,14 +3,15 @@ import json from unittest import TestCase +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_sentry.source import SourceSentry class TestEvents(TestCase): @@ -29,12 +30,8 @@ def state(self): @HttpMocker() def test_read(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() @@ -47,12 +44,8 @@ def test_read(self, http_mocker: HttpMocker): @HttpMocker() def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py index 0103ab4856e8c..40c869236242e 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py @@ -3,14 +3,15 @@ import json from unittest import TestCase +from config_builder import ConfigBuilder +from source_sentry.source import SourceSentry + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_sentry.source import SourceSentry class TestEvents(TestCase): @@ -31,10 +32,9 @@ def test_read(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) # https://sentry.io/api/1/projects/airbyte-09/airbyte-09/issues/?query=lastSeen%3A%3E2022-01-01T00%3A00%3A00.0Z config = self.config() @@ -50,10 +50,9 @@ def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 5f3c1f349ebd8..0bab2123c0fdb 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -5,9 +5,11 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import SyncMode from source_sentry import SourceSentry +from airbyte_cdk.models import SyncMode + + INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} @@ -24,7 +26,10 @@ def test_next_page_token(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": "next-page"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == "next-page" + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == "next-page" + ) def test_next_page_token_is_none(): @@ -33,7 +38,9 @@ def test_next_page_token_is_none(): response_mock.headers = {} # stop condition: "results": "false" response_mock.links = {"next": {"cursor": "", "results": "false"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + ) def test_events_path(): @@ -77,7 +84,10 @@ def test_projects_request_params(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": expected}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == expected + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == expected + ) def test_project_detail_request_params(): @@ -89,10 +99,7 @@ def test_project_detail_request_params(): def test_project_detail_parse_response(requests_mock): expected = {"id": "1", "name": "test project"} stream = get_stream_by_name("project_detail") - requests_mock.get( - "https://sentry.io/api/0/projects/test-org/test-project/", - json=expected - ) + requests_mock.get("https://sentry.io/api/0/projects/test-org/test-project/", json=expected) result = list(stream.read_records(sync_mode=SyncMode.full_refresh))[0] assert expected == result.data @@ -137,4 +144,3 @@ def test_issues_validate_state_value(state, expected): stream = get_stream_by_name("issues") stream.retriever.state = state assert stream.state.get(stream.cursor_field) == expected - diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py index d2f090c60286b..75e813d64e9ad 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/acceptance.py @@ -11,6 +11,7 @@ import docker import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) TMP_FOLDER = "/tmp/test_sftp_source" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py index 5278db267df52..fb90225ddeb12 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/conftest.py @@ -14,10 +14,12 @@ import docker import paramiko import pytest + from airbyte_cdk import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from .utils import get_docker_ip, load_config + logger = logging.getLogger("airbyte") PRIVATE_KEY = str() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py index f6e0b7560c29c..1eb360614d763 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py @@ -10,10 +10,12 @@ from unittest.mock import ANY import pytest +from source_sftp_bulk import SourceSFTPBulk + from airbyte_cdk import AirbyteTracedException, ConfiguredAirbyteCatalog, Status from airbyte_cdk.sources.declarative.models import FailureType from airbyte_cdk.test.entrypoint_wrapper import read -from source_sftp_bulk import SourceSFTPBulk + logger = logging.getLogger("airbyte") @@ -99,18 +101,26 @@ def test_get_files_empty_files(configured_catalog: ConfiguredAirbyteCatalog, con def test_get_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_file_transfer: Mapping[str, Any]): source = SourceSFTPBulk(catalog=configured_catalog, config=config_fixture_use_file_transfer, state=None) output = read(source=source, config=config_fixture_use_file_transfer, catalog=configured_catalog) - expected_file_data = {'bytes': 46_754_266, 'file_relative_path': 'files/file_transfer/file_transfer_1.csv', 'file_url': '/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv', 'modified': ANY, 'source_file_url': '/files/file_transfer/file_transfer_1.csv'} + expected_file_data = { + "bytes": 46_754_266, + "file_relative_path": "files/file_transfer/file_transfer_1.csv", + "file_url": "/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv", + "modified": ANY, + "source_file_url": "/files/file_transfer/file_transfer_1.csv", + } assert len(output.records) == 1 assert list(map(lambda record: record.record.file, output.records)) == [expected_file_data] # Additional assertion to check if the file exists at the file_url path - file_path = expected_file_data['file_url'] + file_path = expected_file_data["file_url"] assert os.path.exists(file_path), f"File not found at path: {file_path}" @pytest.mark.slow @pytest.mark.limit_memory("10 MB") -def test_get_all_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any]): +def test_get_all_file_csv_file_transfer( + configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any] +): """ - The Paramiko dependency `get` method uses requests parallelization for efficiency, which may slightly increase memory usage. - The test asserts that this memory increase remains below the files sizes being transferred. diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py index db92623c29984..390cb801cdb7a 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/utils.py @@ -7,6 +7,7 @@ import re from typing import Any, Mapping, Union + logger = logging.getLogger("airbyte") TMP_FOLDER = "/tmp/test_sftp_source" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/main.py b/airbyte-integrations/connectors/source-sftp-bulk/main.py index 9129b1995968e..f2799e1acdf7f 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/main.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/main.py @@ -5,5 +5,6 @@ from source_sftp_bulk.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py index 8a18fac42f1e9..1f6e04360fc1b 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/client.py @@ -9,9 +9,11 @@ import backoff import paramiko -from airbyte_cdk import AirbyteTracedException, FailureType from paramiko.ssh_exception import AuthenticationException +from airbyte_cdk import AirbyteTracedException, FailureType + + # set default timeout to 300 seconds REQUEST_TIMEOUT = 300 diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py index 96b84eda4c944..5a10c33cfc9d1 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/source.py @@ -12,6 +12,7 @@ from source_sftp_bulk.spec import SourceSFTPBulkSpec from source_sftp_bulk.stream_reader import SourceSFTPBulkStreamReader + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py index 06863da1ae019..23edbcf1ebf07 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/spec.py @@ -3,9 +3,10 @@ from typing import Literal, Optional, Union +from pydantic.v1 import BaseModel, Field + from airbyte_cdk import OneOfOptionConfig from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec, DeliverRawFiles, DeliverRecords -from pydantic.v1 import BaseModel, Field class PasswordCredentials(BaseModel): diff --git a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py index 10d075f001e61..d3f0ae7c525ae 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/source_sftp_bulk/stream_reader.py @@ -9,13 +9,14 @@ from typing import Dict, Iterable, List, Optional import psutil +from typing_extensions import override + from airbyte_cdk import FailureType from airbyte_cdk.sources.file_based.exceptions import FileSizeLimitError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from source_sftp_bulk.client import SFTPClient from source_sftp_bulk.spec import SourceSFTPBulkSpec -from typing_extensions import override class SourceSFTPBulkStreamReader(AbstractFileBasedStreamReader): diff --git a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py index 1fa424d9d1a80..92c81a790b047 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/unit_tests/stream_reader_test.py @@ -10,6 +10,7 @@ from source_sftp_bulk.spec import SourceSFTPBulkSpec from source_sftp_bulk.stream_reader import SourceSFTPBulkStreamReader + logger = logging.Logger("") diff --git a/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sftp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-shopify/main.py b/airbyte-integrations/connectors/source-shopify/main.py index aca13eebbb253..2bfe3cb0cfe99 100644 --- a/airbyte-integrations/connectors/source-shopify/main.py +++ b/airbyte-integrations/connectors/source-shopify/main.py @@ -5,5 +5,6 @@ from source_shopify.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py index 6d53197ff18a8..3d955fd8b5ec2 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -25,7 +25,6 @@ def __init__(self, auth_method: str = None): class ShopifyAuthenticator(TokenAuthenticator): - """ Making Authenticator to be able to accept Header-Based authentication. """ diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py b/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py index eba59a0793555..412c0e3b05a3e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/config_migrations.py @@ -5,12 +5,13 @@ from typing import Any, List, Mapping +from orjson import orjson + from airbyte_cdk.config_observation import create_connector_config_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.models import AirbyteMessageSerializer from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository -from orjson import orjson class MigrateConfig: diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py index d4e4f6b728dcd..873432ef30843 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/http_request.py @@ -3,9 +3,11 @@ from typing import Optional, Union import requests +from requests import exceptions + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction -from requests import exceptions + RESPONSE_CONSUMPTION_EXCEPTIONS = ( exceptions.ChunkedEncodingError, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py index 1a493119b9018..33c1340133c40 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/scopes.py @@ -7,12 +7,14 @@ from typing import Any, Iterable, List, Mapping, Optional import requests -from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import InvalidURL, JSONDecodeError +from airbyte_cdk.sources.streams.http import HttpClient + from .http_request import ShopifyErrorHandler from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyWrongShopNameError + SCOPES_MAPPING: Mapping[str, set[str]] = { # SCOPE: read_customers "Customers": ("read_customers",), @@ -82,7 +84,6 @@ class ShopifyScopes: - # define default logger logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py index a479bf68ff5ad..0034d87ae20b6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py @@ -9,11 +9,12 @@ import pendulum as pdm import requests -from airbyte_cdk.sources.streams.http import HttpClient from requests.exceptions import JSONDecodeError from source_shopify.utils import LOGGER, ApiTypeEnum from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.sources.streams.http import HttpClient + from .exceptions import AirbyteTracedException, ShopifyBulkExceptions from .query import ShopifyBulkQuery, ShopifyBulkTemplates from .record import ShopifyBulkRecord diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py index 7c249196d9283..a1bfcc6766458 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py @@ -2562,7 +2562,6 @@ def _should_include_presentment_prices(self) -> bool: @property def query_nodes(self) -> Optional[Union[List[Field], List[str]]]: - prices_fields: List[str] = ["amount", "currencyCode"] presentment_prices_fields: List[Field] = [ Field( diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py index 61e80937a3542..ffa0852de7999 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/retry.py @@ -8,6 +8,7 @@ from .exceptions import ShopifyBulkExceptions + BULK_RETRY_ERRORS: Final[Tuple] = ( ShopifyBulkExceptions.BulkJobBadResponse, ShopifyBulkExceptions.BulkJobError, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py index dfa3fafdd0c66..40fa1e8b2733d 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py @@ -11,6 +11,7 @@ from .exceptions import ShopifyBulkExceptions + # default end line tag END_OF_FILE: str = "" BULK_PARENT_KEY: str = "__parentId" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py index 462ad3ea3aa8b..d7a7f6b4a76e5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py @@ -9,6 +9,7 @@ from . import schema + _schema = schema _schema_root = _schema.shopify_schema diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py index d3647a5620840..fb24170f11be5 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py @@ -7,6 +7,7 @@ import sgqlc.types.datetime import sgqlc.types.relay + shopify_schema = sgqlc.types.Schema() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 3d420b3318919..76a2e27f83c1c 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -6,11 +6,12 @@ import logging from typing import Any, List, Mapping, Tuple +from requests.exceptions import ConnectionError, RequestException, SSLError + from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from requests.exceptions import ConnectionError, RequestException, SSLError from .auth import MissingAccessTokenError, ShopifyAuthenticator from .scopes import ShopifyScopes diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py index 3837212ad9f22..fc5ead602a7cf 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py @@ -12,11 +12,6 @@ import pendulum as pdm import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpClient, HttpStream -from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler -from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from requests.exceptions import RequestException from source_shopify.http_request import ShopifyErrorHandler from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager @@ -26,6 +21,12 @@ from source_shopify.utils import ShopifyNonRetryableErrors from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.streams.http import HttpClient, HttpStream +from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler +from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING + class ShopifyStream(HttpStream, ABC): # define default logger diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py index 2751a3ab9756c..58f4986221c6e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py @@ -6,8 +6,6 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from requests.exceptions import RequestException from source_shopify.shopify_graphql.bulk.query import ( Collection, @@ -36,6 +34,9 @@ from source_shopify.utils import ApiTypeEnum from source_shopify.utils import ShopifyRateLimiter as limiter +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader + from .base_streams import ( IncrementalShopifyGraphQlBulkStream, IncrementalShopifyNestedStream, @@ -249,7 +250,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream): class BalanceTransactions(IncrementalShopifyStream): - """ PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: https://shopify.dev/api/admin-rest/2021-07/resources/transactions diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index d7fd17a428464..fa510b87ba915 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -10,10 +10,12 @@ from typing import Any, Callable, Dict, Final, List, Mapping, Optional import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction from airbyte_cdk.utils import AirbyteTracedException + # default logger instance LOGGER: Final[logging.Logger] = logging.getLogger("airbyte") @@ -47,7 +49,7 @@ def __new__(self, stream: str) -> Mapping[str, Any]: response_action=ResponseAction.IGNORE, failure_type=FailureType.config_error, error_message=f"Stream `{stream}`. Entity might not be available or missing.", - ) + ), # extend the mapping with more handable errors, if needed. } diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index 0f296c87061a3..4dbba74a333b2 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -9,8 +9,10 @@ import pytest import requests + from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" @@ -39,7 +41,7 @@ def logger(): @pytest.fixture def basic_config(): return { - "shop": "test_shop", + "shop": "test_shop", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "shop_id": 0, } @@ -52,7 +54,6 @@ def auth_config(): "start_date": "2023-01-01", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "authenticator": None, - } @@ -358,7 +359,7 @@ def bulk_job_failed_response(): }, } - + @pytest.fixture def bulk_job_failed_with_partial_url_response(): return { @@ -371,20 +372,16 @@ def bulk_job_failed_with_partial_url_response(): "fileSize": None, "url": None, "partialDataUrl": 'https://some_url?response-content-disposition=attachment;+filename="bulk-123456789.jsonl";+filename*=UTF-8' - "bulk-123456789.jsonl&response-content-type=application/jsonl" + "bulk-123456789.jsonl&response-content-type=application/jsonl", } }, "extensions": { "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 20000.0, - "currentlyAvailable": 19999, - "restoreRate": 1000.0 - } + "throttleStatus": {"maximumAvailable": 20000.0, "currentlyAvailable": 19999, "restoreRate": 1000.0}, } - } + }, } @@ -442,8 +439,8 @@ def bulk_job_running_response(): } }, } - - + + @pytest.fixture def bulk_job_running_with_object_count_and_url_response(): return { diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py index 90eb6faaf67db..afce4d33ab18d 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py @@ -7,7 +7,6 @@ import pytest import requests -from airbyte_cdk.models import SyncMode from source_shopify.shopify_graphql.bulk.exceptions import ShopifyBulkExceptions from source_shopify.shopify_graphql.bulk.status import ShopifyBulkJobStatus from source_shopify.streams.streams import ( @@ -25,15 +24,18 @@ TransactionsGraphql, ) +from airbyte_cdk.models import SyncMode + + _ANY_SLICE = {} _ANY_FILTER_FIELD = "any_filter_field" def test_job_manager_default_values(auth_config) -> None: stream = Products(auth_config) - + # 10Mb chunk size to save the file - assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 + assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 assert stream.job_manager._job_max_retries == 6 assert stream.job_manager._job_backoff_time == 5 # running job logger constrain, every 100-ish message will be printed @@ -48,7 +50,7 @@ def test_job_manager_default_values(auth_config) -> None: # currents: _job_id, _job_state, _job_created_at, _job_self_canceled assert not stream.job_manager._job_id # this string is based on ShopifyBulkJobStatus - assert not stream.job_manager._job_state + assert not stream.job_manager._job_state # completed and saved Bulk Job result filename assert not stream.job_manager._job_result_filename # date-time when the Bulk Job was created on the server @@ -56,7 +58,7 @@ def test_job_manager_default_values(auth_config) -> None: # indicated whether or not we manually force-cancel the current job assert not stream.job_manager._job_self_canceled # time between job status checks - assert stream.job_manager. _job_check_interval == 3 + assert stream.job_manager._job_check_interval == 3 # 0.1 ~= P2H, default value, lower boundary for slice size assert stream.job_manager._job_size_min == 0.1 # last running job object count @@ -99,8 +101,9 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: {"json": request.getfixturevalue("bulk_error_with_concurrent_job")}, # concurrent request has finished {"json": request.getfixturevalue("bulk_successful_response")}, - ]) - + ], + ) + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) # call count should be 4 (3 retries, 1 - succeeded) assert requests_mock.call_count == 4 @@ -119,14 +122,16 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: ], ids=[ "max attempt reached", - ] + ], ) -def test_job_retry_on_concurrency(request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected) -> None: +def test_job_retry_on_concurrency( + request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # patching concurrent settings stream.job_manager._concurrent_max_retry = concurrent_max_retry stream.job_manager._concurrent_interval = 1 - + requests_mock.post(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) if error_type: @@ -200,8 +205,8 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, mocker.patch("source_shopify.shopify_graphql.bulk.record.ShopifyBulkRecord.read_file", return_value=[]) stream.job_manager._job_check_state() assert expected == stream.job_manager._job_result_filename - - + + @pytest.mark.parametrize( "job_response, error_type, expected", [ @@ -211,7 +216,9 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, "failed", ], ) -def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, requests_mock, job_response, error_type, expected, auth_config) -> None: +def test_job_failed_for_stream_with_no_bulk_checkpointing( + mocker, request, requests_mock, job_response, error_type, expected, auth_config +) -> None: stream = InventoryLevels(auth_config) # modify the sleep time for the test stream.job_manager._concurrent_max_retry = 1 @@ -253,26 +260,28 @@ def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, reque "BulkJobBadResponse", ], ) -def test_retry_on_job_creation_exception(request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg) -> None: +def test_retry_on_job_creation_exception( + request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg +) -> None: stream = MetafieldOrders(auth_config) stream.job_manager._job_backoff_time = 0 stream.job_manager._job_max_retries = max_retry # patching the method to get the right ID checks if job_response: stream.job_manager._job_id = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("id") - + if job_state: # setting job_state to simulate the error-in-the-middle stream.job_manager._job_state = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("status") - + # mocking the response for STATUS CHECKS json_mock_response = request.getfixturevalue(job_response) if job_response else None requests_mock.post(stream.job_manager.base_url, json=json_mock_response) - + # testing raised exception and backoff with pytest.raises(error_type) as error: stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) - + # we expect different call_count, because we set the different max_retries assert expected_msg in repr(error.value) and requests_mock.call_count == call_count_expected @@ -302,11 +311,11 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a job_result_url = test_job_status_response.json().get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + # mocking the nested request call to retrieve the data from result URL stream.job_manager._job_id = job_id requests_mock.get(job_result_url, json=request.getfixturevalue(job_response)) - + # calling the sceario processing stream.job_manager._job_track_running() assert stream.job_manager._job_state == expected @@ -316,13 +325,13 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "running_job_response, canceled_job_response, expected", [ ( - "bulk_job_running_with_object_count_and_url_response", - "bulk_job_canceled_with_object_count_and_url_response", + "bulk_job_running_with_object_count_and_url_response", + "bulk_job_canceled_with_object_count_and_url_response", "bulk-123456789.jsonl", ), ( - "bulk_job_running_with_object_count_no_url_response", - "bulk_job_canceled_with_object_count_no_url_response", + "bulk_job_running_with_object_count_no_url_response", + "bulk_job_canceled_with_object_count_no_url_response", None, ), ], @@ -331,7 +340,9 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "self-canceled with no url", ], ) -def test_job_running_with_canceled_scenario(mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected) -> None: +def test_job_running_with_canceled_scenario( + mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # modify the sleep time for the test stream.job_manager._job_check_interval = 0 @@ -339,7 +350,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_id = request.getfixturevalue(running_job_response).get("data", {}).get("node", {}).get("id") # mocking the response for STATUS CHECKS requests_mock.post( - stream.job_manager.base_url, + stream.job_manager.base_url, [ {"json": request.getfixturevalue(running_job_response)}, {"json": request.getfixturevalue(canceled_job_response)}, @@ -348,7 +359,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_result_url = request.getfixturevalue(canceled_job_response).get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + stream.job_manager._job_id = job_id stream.job_manager._job_checkpoint_interval = 5 # faking self-canceled job @@ -448,9 +459,9 @@ def test_bulk_stream_parse_response( ) def test_stream_slices( auth_config, - stream, - stream_state, - with_start_date, + stream, + stream_state, + with_start_date, expected_start, ) -> None: # simulating `None` for `start_date` and `config migration` @@ -462,7 +473,7 @@ def test_stream_slices( test_result = list(stream.stream_slices(stream_state=stream_state)) assert test_result[0].get("start") == expected_start - + @pytest.mark.parametrize( "stream, json_content_example, last_job_elapsed_time, previous_slice_size, adjusted_slice_size", [ @@ -471,7 +482,7 @@ def test_stream_slices( ids=[ "Expand Slice Size", ], -) +) def test_expand_stream_slices_job_size( request, requests_mock, @@ -483,7 +494,6 @@ def test_expand_stream_slices_job_size( adjusted_slice_size, auth_config, ) -> None: - stream = stream(auth_config) # get the mocked job_result_url test_result_url = bulk_job_completed_response.get("data").get("node").get("url") @@ -495,7 +505,7 @@ def test_expand_stream_slices_job_size( # for the sake of simplicity we fake some parts to simulate the `current_job_time_elapsed` # fake current slice interval value stream.job_manager._job_size = previous_slice_size - # fake `last job elapsed time` + # fake `last job elapsed time` if last_job_elapsed_time: stream.job_manager._job_last_elapsed_time = last_job_elapsed_time diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py index 8a5d7012d38bc..cf586b159ae17 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -30,12 +30,12 @@ def test_query_status() -> None: } } }""" - + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.status(input_job_id) assert repr(template) == repr(expected) - - + + def test_bulk_query_prepare() -> None: expected = '''mutation { bulkOperationRunQuery( @@ -54,14 +54,14 @@ def test_bulk_query_prepare() -> None: } } }''' - + input_query_from_slice = "{some_query}" template = ShopifyBulkTemplates.prepare(input_query_from_slice) assert repr(template) == repr(expected) - - + + def test_bulk_query_cancel() -> None: - expected = '''mutation { + expected = """mutation { bulkOperationCancel(id: "gid://shopify/BulkOperation/4047052112061") { bulkOperation { id @@ -73,32 +73,32 @@ def test_bulk_query_cancel() -> None: message } } - }''' - + }""" + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.cancel(input_job_id) assert repr(template) == repr(expected) - + @pytest.mark.parametrize( "query_name, fields, filter_field, start, end, expected", [ ( - "test_root", - ["test_field1", "test_field2"], + "test_root", + ["test_field1", "test_field2"], "updated_at", "2023-01-01", - "2023-01-02", + "2023-01-02", Query( - name='test_root', + name="test_root", arguments=[ - Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=["test_field1", "test_field2"])])] - ) + Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), + ], + fields=[Field(name="edges", fields=[Field(name="node", fields=["test_field1", "test_field2"])])], + ), ) ], - ids=["simple query with filter and sort"] + ids=["simple query with filter and sort"], ) def test_base_build_query(basic_config, query_name, fields, filter_field, start, end, expected) -> None: """ @@ -116,7 +116,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, } ''' """ - + builder = ShopifyBulkQuery(basic_config) filter_query = f"{filter_field}:>'{start}' AND {filter_field}:<='{end}'" built_query = builder.build(query_name, fields, filter_query) @@ -136,14 +136,52 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='customers', + name="customers", arguments=[ Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - Argument(name="sortKey", value="UPDATED_AT"), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=['__typename', 'id', Field(name="updatedAt", alias="customers_updated_at"), Field(name="metafields", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", "namespace", "value", "key", "description", "createdAt", "updatedAt", "type"])])])])])] + Argument(name="sortKey", value="UPDATED_AT"), + ], + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + Field(name="updatedAt", alias="customers_updated_at"), + Field( + name="metafields", + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + "namespace", + "value", + "key", + "description", + "createdAt", + "updatedAt", + "type", + ], + ) + ], + ) + ], + ), + ], + ) + ], + ) + ], ) - ] + ], ), ), ( @@ -206,28 +244,28 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "createdAt", "updatedAt", "type", - ] + ], ) - ] + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] - ) + ], + ), ), ( InventoryLevel, @@ -239,64 +277,91 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='locations', + name="locations", arguments=[ Argument(name="includeLegacy", value="true"), Argument(name="includeInactive", value="true"), - ], + ], fields=[ Field( - name='edges', + name="edges", fields=[ Field( - name='node', + name="node", fields=[ - '__typename', - 'id', + "__typename", + "id", Field( - name="inventoryLevels", + name="inventoryLevels", arguments=[ - Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - ], + Argument( + name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\"" + ), + ], fields=[ Field( - name="edges", + name="edges", fields=[ Field( - name="node", + name="node", fields=[ - "__typename", - "id", - "canDeactivate", - "createdAt", - "deactivationAlert", - "updatedAt", - Field(name="item", fields=[Field(name="inventoryHistoryUrl", alias="inventory_history_url"), Field(name="id", alias="inventory_item_id"), Field(name="locationsCount", alias="locations_count", fields=["count"])]), - Field( - name="quantities", + "__typename", + "id", + "canDeactivate", + "createdAt", + "deactivationAlert", + "updatedAt", + Field( + name="item", + fields=[ + Field( + name="inventoryHistoryUrl", alias="inventory_history_url" + ), + Field(name="id", alias="inventory_item_id"), + Field( + name="locationsCount", + alias="locations_count", + fields=["count"], + ), + ], + ), + Field( + name="quantities", arguments=[ - Argument(name="names", value=['"available"', '"incoming"', '"committed"', '"damaged"', '"on_hand"', '"quality_control"', '"reserved"', '"safety_stock"']) - ], + Argument( + name="names", + value=[ + '"available"', + '"incoming"', + '"committed"', + '"damaged"', + '"on_hand"', + '"quality_control"', + '"reserved"', + '"safety_stock"', + ], + ) + ], fields=[ "id", "name", "quantity", "updatedAt", ], - ) - ] + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] + ], ), ), ], @@ -304,7 +369,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "MetafieldCustomers query with 1 query_path(str)", "MetafieldProductImages query with composite quey_path(List[2])", "InventoryLevel query", - ] + ], ) def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_stream_class, expected) -> None: if parent_stream_class: @@ -314,4 +379,4 @@ def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_s else: stream_query = query_class(auth_config) - assert stream_query.get(filter_field, start, end) == expected.render() \ No newline at end of file + assert stream_query.get(filter_field, start, end) == expected.render() diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py index dff60ea605d59..85c48ce6d13b3 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py @@ -58,7 +58,7 @@ def test_check_type(basic_config, record, types, expected) -> None: "alias_to_id_field": "gid://shopify/Metafield/123", "__parentId": "gid://shopify/Order/102030", }, - ) + ), ], ) def test_record_resolver(basic_config, record, expected) -> None: diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py index df16077abc14b..d878a1c4e9a4a 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/authentication.py @@ -2,11 +2,13 @@ import json +from source_shopify.scopes import SCOPES_MAPPING +from source_shopify.streams.base_streams import ShopifyStream + from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS from airbyte_cdk.test.mock_http.response_builder import find_template -from source_shopify.scopes import SCOPES_MAPPING -from source_shopify.streams.base_streams import ShopifyStream + _ALL_SCOPES = [scope for stream_scopes in SCOPES_MAPPING.values() for scope in stream_scopes] diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py index 05b15d05dc008..29ae2d9ae93e7 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py @@ -4,10 +4,11 @@ from datetime import datetime from random import randint -from airbyte_cdk.test.mock_http import HttpRequest, HttpResponse from source_shopify.shopify_graphql.bulk.query import ShopifyBulkTemplates from source_shopify.streams.base_streams import ShopifyStream +from airbyte_cdk.test.mock_http import HttpRequest, HttpResponse + def _create_job_url(shop_name: str) -> str: return f"https://{shop_name}.myshopify.com/admin/api/{ShopifyStream.api_version}/graphql.json" @@ -42,16 +43,15 @@ def create_job_creation_body(lower_boundary: datetime, upper_boundary: datetime) } } }""" - query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace("%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat()) + query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace( + "%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat() + ) prepared_query = ShopifyBulkTemplates.prepare(query) return json.dumps({"query": prepared_query}) def create_job_creation_request(shop_name: str, lower_boundary: datetime, upper_boundary: datetime) -> HttpRequest: - return HttpRequest( - url=_create_job_url(shop_name), - body=create_job_creation_body(lower_boundary, upper_boundary) - ) + return HttpRequest(url=_create_job_url(shop_name), body=create_job_creation_body(lower_boundary, upper_boundary)) def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: @@ -70,7 +70,7 @@ def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: partialDataUrl }} }} - }}""" + }}""", ) @@ -89,33 +89,26 @@ def create_job_cancel_request(shop_name: str, job_id: str) -> HttpRequest: message }} }} - }}""" + }}""", ) + class JobCreationResponseBuilder: - def __init__(self, job_created_at:str="2024-05-05T02:00:00Z") -> None: + def __init__(self, job_created_at: str = "2024-05-05T02:00:00Z") -> None: self._template = { "data": { "bulkOperationRunQuery": { - "bulkOperation": { - "id": "gid://shopify/BulkOperation/0", - "status": "CREATED", - "createdAt": f"{job_created_at}" - }, - "userErrors": [] + "bulkOperation": {"id": "gid://shopify/BulkOperation/0", "status": "CREATED", "createdAt": f"{job_created_at}"}, + "userErrors": [], } }, "extensions": { "cost": { "requestedQueryCost": 10, "actualQueryCost": 10, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1990, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1990, "restoreRate": 100.0}, } - } + }, } def with_bulk_operation_id(self, bulk_operation_id: str) -> "JobCreationResponseBuilder": @@ -135,30 +128,26 @@ def __init__(self) -> None: "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1999, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1999, "restoreRate": 100.0}, } - } + }, } } - def with_running_status(self, bulk_operation_id: str, object_count: str="10") -> "JobStatusResponseBuilder": + def with_running_status(self, bulk_operation_id: str, object_count: str = "10") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { - "id": bulk_operation_id, - "status": "RUNNING", - "errorCode": None, - "createdAt": "2024-05-28T18:57:54Z", - "objectCount": object_count, - "fileSize": None, - "url": None, - "partialDataUrl": None, + "id": bulk_operation_id, + "status": "RUNNING", + "errorCode": None, + "createdAt": "2024-05-28T18:57:54Z", + "objectCount": object_count, + "fileSize": None, + "url": None, + "partialDataUrl": None, } return self - def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "COMPLETED", @@ -167,11 +156,11 @@ def with_completed_status(self, bulk_operation_id: str, job_result_url: str, obj "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self - def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "CANCELED", @@ -180,7 +169,7 @@ def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, obje "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self @@ -192,15 +181,14 @@ class MetafieldOrdersJobResponseBuilder: def __init__(self) -> None: self._records = [] - def _any_record(self, updated_at:str="2024-05-05T01:09:50Z") -> str: + def _any_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> str: an_id = str(randint(1000000000000, 9999999999999)) a_parent_id = str(randint(1000000000000, 9999999999999)) return f"""{{"__typename":"Order","id":"gid:\/\/shopify\/Order\/{a_parent_id}"}} {{"__typename":"Metafield","id":"gid:\/\/shopify\/Metafield\/{an_id}","namespace":"my_fields","value":"asdfasdf","key":"purchase_order","description":null,"createdAt":"2023-04-13T12:09:50Z","updatedAt":"{updated_at}","type":"single_line_text_field","__parentId":"gid:\/\/shopify\/Order\/{a_parent_id}"}} """ - - def with_record(self, updated_at:str="2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": + def with_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": self._records.append(self._any_record(updated_at=updated_at)) return self diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py index 8a0d6d5c8c59b..ad8d1e31fb379 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py @@ -5,17 +5,20 @@ from unittest import TestCase import pytest + from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse + _AN_ERROR_RESPONSE = HttpResponse(json.dumps({"errors": ["an error"]})) _SERVICE_UNAVAILABLE_ERROR_RESPONSE = HttpResponse(json.dumps({"errors": ["Service unavailable"]}), status_code=503) -from airbyte_cdk.models import AirbyteStateMessage, SyncMode -from airbyte_cdk.test.state_builder import StateBuilder from freezegun import freeze_time from requests.exceptions import ConnectionError from source_shopify import SourceShopify + +from airbyte_cdk.models import AirbyteStateMessage, SyncMode +from airbyte_cdk.test.state_builder import StateBuilder from unit_tests.integration.api.authentication import grant_all_scopes, set_up_shop from unit_tests.integration.api.bulk import ( JobCreationResponseBuilder, @@ -27,6 +30,7 @@ create_job_status_request, ) + _BULK_OPERATION_ID = "gid://shopify/BulkOperation/4472588009661" _BULK_STREAM = "metafield_orders" _SHOP_NAME = "airbyte-integration-test" @@ -41,6 +45,7 @@ _INCREMENTAL_JOB_START_DATE = datetime.fromisoformat(_INCREMENTAL_JOB_START_DATE_ISO) _INCREMENTAL_JOB_END_DATE = _INCREMENTAL_JOB_START_DATE + timedelta(hours=24, minutes=0) + def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_interval=200000) -> Dict[str, Any]: return { "start_date": start_date.strftime("%Y-%m-%d"), @@ -50,13 +55,12 @@ def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_inter "api_password": "api_password", }, "bulk_window_in_days": bulk_window, - "job_checkpoint_interval": job_checkpoint_interval + "job_checkpoint_interval": job_checkpoint_interval, } @freeze_time(_JOB_END_DATE) class GraphQlBulkStreamTest(TestCase): - def setUp(self) -> None: self._http_mocker = HttpMocker() self._http_mocker.__enter__() @@ -103,8 +107,10 @@ def test_given_response_is_not_json_on_job_creation_when_read_then_retry(self) - job_creation_request, [ HttpResponse("This is not json"), - JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), # This will never get called (see assertion below) - ] + JobCreationResponseBuilder() + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), # This will never get called (see assertion below) + ], ) self._http_mocker.post( @@ -126,8 +132,11 @@ def test_given_connection_error_on_job_creation_when_read_then_retry_job_creatio inner_mocker.register_uri( # TODO the testing library should have the ability to generate ConnectionError. As this might not be trivial, we will wait for another case before implementing "POST", _URL_GRAPHQL, - [{"exc": ConnectionError("ConnectionError")}, {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}], - additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE) + [ + {"exc": ConnectionError("ConnectionError")}, + {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}, + ], + additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -152,7 +161,7 @@ def test_given_retryable_error_on_first_get_job_status_when_read_then_retry(self [ _AN_ERROR_RESPONSE, JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -175,7 +184,7 @@ def test_given_retryable_error_on_get_job_status_when_read_then_retry(self) -> N JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID).build(), HttpResponse(json.dumps({"errors": ["an error"]})), JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -209,7 +218,9 @@ def test_when_read_then_extract_records(self) -> None: job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=5) self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -220,14 +231,10 @@ def test_when_read_then_extract_records(self) -> None: MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), ) # expectation is job start date should be the updated_at in orders - metafield_orders_orders_state = {"orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + metafield_orders_orders_state = { + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago @@ -238,13 +245,13 @@ def test_when_read_then_extract_records(self) -> None: assert len(output.records) == 2 def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(self) -> None: - """" + """ " The motivation of this test is https://github.com/airbytehq/oncall/issues/6874 - + In this scenario we end having stream_slices method to generate same slice N times. Our checkpointing logic will trigger when job_checkpoint_interval is passed, but there may be the case that such checkpoint has the same value as the current slice start date so we would end requesting same job. - + In this test: 1. First job requires to checkpoint as we pass the 1500 limit, it cancels the bulk job and checkpoints from last cursor value. 2. Next job just goes "fine". @@ -261,6 +268,7 @@ def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(s {"type":"LOG","log":{"level":"INFO","message":"Stream: `metafield_orders`, the BULK Job: `gid://shopify/BulkOperation/4472588009771` is CREATED"}} ... """ + def add_n_records(builder, n, record_date: Optional[str] = None): for _ in range(n): builder = builder.with_record(updated_at=record_date) @@ -271,7 +279,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # create a job request self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) # get job status self._http_mocker.post( @@ -282,15 +292,10 @@ def add_n_records(builder, n, record_date: Optional[str] = None): JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID, object_count="16000").build(), # this will complete the job JobStatusResponseBuilder().with_canceled_status(_BULK_OPERATION_ID, _JOB_RESULT_URL, object_count="1700").build(), - ] + ], ) # mock the cancel operation request as we passed the 15000 rows - self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), - [ - HttpResponse(json.dumps({}), status_code=200) - ] - ) + self._http_mocker.post(create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), [HttpResponse(json.dumps({}), status_code=200)]) # get results for the request that got cancelled adjusted_checkpoint_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(days=2, hours=6, minutes=30) adjusted_record_date = adjusted_checkpoint_start_date.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -309,7 +314,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): self._http_mocker.post( # The start date is caused by record date in previous iteration create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693950.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693950.jsonl&response-content-type=application%2Fjsonl" @@ -318,7 +325,7 @@ def add_n_records(builder, n, record_date: Optional[str] = None): [ # this will output the job is running JobStatusResponseBuilder().with_completed_status(next_bulk_operation_id, next_job_result_url).build(), - ] + ], ) # get results for the request that got cancelled self._http_mocker.get( @@ -333,12 +340,14 @@ def add_n_records(builder, n, record_date: Optional[str] = None): adjusted_checkpoint_end_date = adjusted_checkpoint_start_date + timedelta(days=1) job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=4) create_job_request = create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date) - + self._http_mocker.post( create_job_request, - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) - + base_status_responses = [ JobStatusResponseBuilder().with_running_status(next_bulk_operation_id, object_count="500").build(), # this should make the job get canceled as it gets over 15000 rows @@ -346,23 +355,17 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # this will complete the job JobStatusResponseBuilder().with_canceled_status(next_bulk_operation_id, next_job_result_url, object_count="1700").build(), ] - + n_times_to_loop = 4 responses_in_loop = base_status_responses * n_times_to_loop # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693960.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693960.jsonl&response-content-type=application%2Fjsonl" - - self._http_mocker.post( - create_job_status_request(_SHOP_NAME, next_bulk_operation_id), - responses_in_loop - ) + + self._http_mocker.post(create_job_status_request(_SHOP_NAME, next_bulk_operation_id), responses_in_loop) # mock the cancel operation request as we passed the 15000 rows self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), - [ - HttpResponse(json.dumps({}), status_code=200) - ] + create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), [HttpResponse(json.dumps({}), status_code=200)] ) # get results @@ -371,35 +374,28 @@ def add_n_records(builder, n, record_date: Optional[str] = None): HttpRequest(next_job_result_url), add_n_records(MetafieldOrdersJobResponseBuilder(), 80, adjusted_record_date).build(), ) - + # ********* end of request mocking ************* metafield_orders_orders_state = { - "orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago config_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(weeks=104) - output = self._read(_get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state) + output = self._read( + _get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state + ) expected_error_message = "The stream: `metafield_orders` checkpoint collision is detected." result = output.errors[0].trace.error.internal_message - # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` + # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` assert result is not None and expected_error_message in result - def _read(self, config, sync_mode=SyncMode.full_refresh, state: Optional[List[AirbyteStateMessage]] = None): catalog = CatalogBuilder().with_stream(_BULK_STREAM, sync_mode).build() output = read(SourceShopify(), config, catalog, state=state) return output - - - diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py index ee1ae56a74a17..932fc7a33fa71 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py @@ -7,6 +7,7 @@ from source_shopify.auth import NotImplementedAuth, ShopifyAuthenticator from source_shopify.source import ConnectionCheckTest + TEST_ACCESS_TOKEN = "test_access_token" TEST_API_PASSWORD = "test_api_password" diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py index 80cc6115c4490..718ebb2537c27 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py @@ -7,6 +7,7 @@ from source_shopify.streams.streams import OrderRefunds, Orders from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache + # Define the Stream instances for the tests SHOPIFY_STREAM = Orders(config={"authenticator": None}) SHOPIFY_SUB_STREAM = OrderRefunds(config={"authenticator": None}) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py index 49850a1ef8a18..daedb2a89a9d0 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py @@ -6,6 +6,7 @@ import requests from source_shopify.utils import ShopifyRateLimiter as limiter + TEST_DATA_FIELD = "some_data_field" TEST_RATE_LIMIT_HEADER = "X-Shopify-Shop-Api-Call-Limit" TEST_THRESHOLD = 0.9 diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py index 3d599ce770b11..ea902f0be5676 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py @@ -108,7 +108,9 @@ def test_read_deleted_records(stream, requests_mock, deleted_records_json, expec stream = stream(config) deleted_records_url = stream.url_base + stream.deleted_events.path() requests_mock.get(deleted_records_url, json=deleted_records_json) - mocker.patch("source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json) + mocker.patch( + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json + ) assert list(stream.read_records(sync_mode=None)) == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py index a6c99f9c3c44b..285b816734ed4 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py @@ -7,10 +7,25 @@ @pytest.mark.parametrize( "page_size, filter_value, next_page_token, expected_query", [ - (100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + ( + 100, + None, + None, + "query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 200, + "2027-07-11T13:07:45-07:00", + None, + "query {\n products(first: 200, query: \"updated_at:>'2027-07-11T13:07:45-07:00'\", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 250, + "2027-07-11T13:07:45-07:00", + "end_cursor_value", + 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}', + ), ], ) def test_get_query_products(page_size, filter_value, next_page_token, expected_query): - assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query + assert get_query_products(page_size, "updatedAt", filter_value, next_page_token) == expected_query diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py index de54e242294a4..c90b81052dda6 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_migrations/test_config_migrations.py @@ -6,11 +6,13 @@ import json from typing import Any, Mapping -from airbyte_cdk.models import OrchestratorType, Type -from airbyte_cdk.sources import Source from source_shopify.config_migrations import MigrateConfig from source_shopify.source import SourceShopify +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source + + # BASE ARGS CMD = "check" TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index f9ef2ba2f7cad..797be1e385f48 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.utils import AirbyteTracedException from source_shopify.auth import ShopifyAuthenticator from source_shopify.source import ConnectionCheckTest, SourceShopify from source_shopify.streams.streams import ( @@ -49,6 +48,8 @@ TransactionsGraphql, ) +from airbyte_cdk.utils import AirbyteTracedException + from .conftest import records_per_slice @@ -126,13 +127,13 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi else: result = stream.path() assert result == expected_path - - + + @pytest.mark.parametrize( "stream, parent_records, state_checkpoint_interval", [ ( - OrderRefunds, + OrderRefunds, [ {"id": 1, "refunds": [{"created_at": "2021-01-01T00:00:00+00:00"}]}, {"id": 2, "refunds": [{"created_at": "2021-02-01T00:00:00+00:00"}]}, @@ -145,10 +146,10 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi ], ) def test_stream_slice_nested_substream_buffering( - mocker, - config, - stream, - parent_records, + mocker, + config, + stream, + parent_records, state_checkpoint_interval, ) -> None: # making the stream instance @@ -156,7 +157,7 @@ def test_stream_slice_nested_substream_buffering( stream.state_checkpoint_interval = state_checkpoint_interval # simulating `read_records` for the `parent_stream` mocker.patch( - "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=parent_records, ) # count how many slices we expect, based on the number of parent_records @@ -173,7 +174,7 @@ def test_stream_slice_nested_substream_buffering( # count total slices total_slices += 1 # check we have emitted complete number of slices - assert total_slices == total_slices_expected + assert total_slices == total_slices_expected def test_check_connection(config, mocker) -> None: @@ -212,11 +213,23 @@ def test_request_params(config, stream, expected) -> None: "last_record, current_state, expected", [ # no init state - ({"created_at": "2022-10-10T06:21:53-07:00"}, {}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is empty str - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": ""}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": ""}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is None - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": None}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": None}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # last rec cursor is None ({"created_at": None}, {"created_at": None}, {"created_at": "", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), # last rec cursor is empty str @@ -257,21 +270,19 @@ def test_get_shop_name(config, shop, expected) -> None: actual = source.get_shop_name(config) assert actual == expected + @pytest.mark.parametrize( "config, expected_stream_class", [ ({"fetch_transactions_user_id": False}, TransactionsGraphql), ({"fetch_transactions_user_id": True}, Transactions), ({}, TransactionsGraphql), - ], + ], ids=["don't fetch user_id", "fetch user id", "unset config value shouldn't fetch user_id"], ) def test_select_transactions_stream(config, expected_stream_class): config["shop"] = "test-store" - config["credentials"] = { - "auth_method": "api_password", - "api_password": "shppa_123" - } + config["credentials"] = {"auth_method": "api_password", "api_password": "shppa_123"} config["authenticator"] = ShopifyAuthenticator(config) source = SourceShopify() @@ -284,7 +295,7 @@ def test_select_transactions_stream(config, expected_stream_class): [ pytest.param([{"id": "12345"}], "12345", None, id="test_shop_name_exists"), pytest.param([], None, AirbyteTracedException, id="test_shop_name_does_not_exist"), - ], + ], ) def test_get_shop_id(config, read_records, expected_shop_id, expected_error): check_test = ConnectionCheckTest(config) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index ca9e306e66983..649e409e70561 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -100,8 +100,9 @@ def test_privileges_validation(requests_mock, fetch_transactions_user_id, basic_ "Internal Server Error for slice (500)", ], ) -def test_unavailable_stream(requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, - json_response: Mapping[str, Any]): +def test_unavailable_stream( + requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, json_response: Mapping[str, Any] +): stream = stream(auth_config) url = stream.url_base + stream.path(stream_slice=slice) requests_mock.get(url=url, json=json_response, status_code=status_code) diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py index d49b558823338..a9256a5339721 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-singlestore/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-slack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-slack/main.py b/airbyte-integrations/connectors/source-slack/main.py index b2ff9c8511630..f80fc7bb2ca11 100644 --- a/airbyte-integrations/connectors/source-slack/main.py +++ b/airbyte-integrations/connectors/source-slack/main.py @@ -4,5 +4,6 @@ from source_slack.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py index 9dbb401a07e9d..0ee57c4d76ad1 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py @@ -4,6 +4,7 @@ from typing import List import requests + from airbyte_cdk.sources.declarative.extractors import DpathExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py index 02bd7ee308669..ec353d4921a19 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py @@ -5,6 +5,7 @@ from typing import Any, Iterable, List, Mapping, Optional import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever @@ -13,6 +14,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + LOGGER = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py b/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py index 0db32e4bce139..cba204c928bd2 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/slack_backoff_strategy.py @@ -4,9 +4,10 @@ import logging from typing import Optional, Union -from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy from requests import RequestException, Response +from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy + class SlackBackoffStrategy(BackoffStrategy): def __init__(self, logger: logging.Logger): diff --git a/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py index cc6d9cd036070..aabfeadd3c7e2 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py @@ -8,6 +8,7 @@ from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from source_slack import SourceSlack + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 5328556a1624f..bf600c7a6bc94 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping import pendulum + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream diff --git a/airbyte-integrations/connectors/source-slack/source_slack/streams.py b/airbyte-integrations/connectors/source-slack/source_slack/streams.py index 1530c74a0ae54..8623af137fef4 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/streams.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/streams.py @@ -8,12 +8,13 @@ import pendulum import requests +from pendulum import DateTime + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.error_handlers import BackoffStrategy, ErrorHandler, HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING -from pendulum import DateTime from source_slack.components.slack_backoff_strategy import SlackBackoffStrategy from .components.join_channels import JoinChannelsStream diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 002a9ec96779d..3d03f4755646a 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -8,6 +8,7 @@ import pytest + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" @@ -16,13 +17,10 @@ def conversations_list(requests_mock): return requests_mock.register_uri( "GET", "https://slack.com/api/conversations.list?limit=1000&types=public_channel", - json={ - "channels": [ - {"id": "airbyte-for-beginners", "is_member": True}, - {"id": "good-reads", "is_member": True}] - }, + json={"channels": [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}]}, ) + def base_config() -> MutableMapping: return copy.deepcopy( { @@ -98,11 +96,27 @@ def invalid_config() -> MutableMapping: @pytest.fixture def joined_channel(): - return {"id": "C061EG9SL", "name": "general", "is_channel": True, "is_group": False, "is_im": False, - "created": 1449252889, - "creator": "U061F7AUR", "is_archived": False, "is_general": True, "unlinked": 0, "name_normalized": "general", - "is_shared": False, - "is_ext_shared": False, "is_org_shared": False, "pending_shared": [], "is_pending_ext_shared": False, - "is_member": True, "is_private": False, "is_mpim": False, - "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, - "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, "previous_names": []} + return { + "id": "C061EG9SL", + "name": "general", + "is_channel": True, + "is_group": False, + "is_im": False, + "created": 1449252889, + "creator": "U061F7AUR", + "is_archived": False, + "is_general": True, + "unlinked": 0, + "name_normalized": "general", + "is_shared": False, + "is_ext_shared": False, + "is_org_shared": False, + "pending_shared": [], + "is_pending_ext_shared": False, + "is_member": True, + "is_private": False, + "is_mpim": False, + "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, + "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, + "previous_names": [], + } diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py index dc21220a01391..d0068d1f4aa9e 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py @@ -4,13 +4,14 @@ import pendulum import pytest +from source_slack import SourceSlack +from source_slack.components.channel_members_extractor import ChannelMembersExtractor +from source_slack.components.join_channels import ChannelsRetriever, JoinChannelsStream + from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordSelector from airbyte_cdk.sources.declarative.requesters import HttpRequester from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_protocol.models import SyncMode -from source_slack import SourceSlack -from source_slack.components.channel_members_extractor import ChannelMembersExtractor -from source_slack.components.join_channels import ChannelsRetriever, JoinChannelsStream def get_stream_by_name(stream_name, config): @@ -23,22 +24,13 @@ def get_stream_by_name(stream_name, config): def test_channel_members_extractor(token_config): response_mock = MagicMock() - response_mock.json.return_value = {"members": [ - "U023BECGF", - "U061F7AUR", - "W012A3CDE" - ]} + response_mock.json.return_value = {"members": ["U023BECGF", "U061F7AUR", "W012A3CDE"]} records = ChannelMembersExtractor(config=token_config, parameters={}, field_path=["members"]).extract_records(response=response_mock) - assert records == [{"member_id": "U023BECGF"}, - {"member_id": "U061F7AUR"}, - {"member_id": "W012A3CDE"}] + assert records == [{"member_id": "U023BECGF"}, {"member_id": "U061F7AUR"}, {"member_id": "W012A3CDE"}] def test_join_channels(token_config, requests_mock, joined_channel): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json={"ok": True, "channel": joined_channel} - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json={"ok": True, "channel": joined_channel}) token = token_config["credentials"]["api_token"] authenticator = TokenAuthenticator(token) channel_filter = token_config["channel_filter"] @@ -51,13 +43,16 @@ def test_join_channels(token_config, requests_mock, joined_channel): def get_channels_retriever_instance(token_config): return ChannelsRetriever( config=token_config, - requester=HttpRequester(name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, - parameters={}), + requester=HttpRequester( + name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, parameters={} + ), record_selector=RecordSelector( extractor=DpathExtractor(field_path=["channels"], config=token_config, parameters={}), - config=token_config, parameters={}, - schema_normalization=None), - parameters={} + config=token_config, + parameters={}, + schema_normalization=None, + ), + parameters={}, ) @@ -76,20 +71,22 @@ def test_join_channels_make_join_channel_slice(token_config): @pytest.mark.parametrize( "join_response, log_message", ( - ({"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, "Successfully joined channel: test channel"), - ({"ok": False, "error": "missing_scope", "needed": "channels:write"}, - "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}"), + ( + {"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, + "Successfully joined channel: test channel", + ), + ( + {"ok": False, "error": "missing_scope", "needed": "channels:write"}, + "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}", + ), ), - ids=["successful_join_to_channel", "failed_join_to_channel"] + ids=["successful_join_to_channel", "failed_join_to_channel"], ) def test_join_channel_read(requests_mock, token_config, joined_channel, caplog, join_response, log_message): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json=join_response - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json=join_response) requests_mock.get( url="https://slack.com/api/conversations.list", - json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]} + json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]}, ) retriever = get_channels_retriever_instance(token_config) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py index 761597a66fc22..7ceae8c61119c 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py @@ -7,6 +7,7 @@ from source_slack import SourceSlack from source_slack.config_migrations import MigrateLegacyConfig + CMD = "check" TEST_CONFIG_LEGACY_PATH = f"{os.path.dirname(__file__)}/configs/legacy_config.json" TEST_CONFIG_ACTUAL_PATH = f"{os.path.dirname(__file__)}/configs/actual_config.json" diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index ae1a589227970..90f6531ca2582 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -17,6 +17,7 @@ def get_stream_by_name(stream_name, config): return stream raise ValueError(f"Stream {stream_name} not found") + @parametrized_configs def test_streams(conversations_list, config, is_valid): source = SourceSlack() diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index b9966eb3594c1..4ec3b24c517a0 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -6,12 +6,13 @@ import pendulum import pytest -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from requests import Response from source_slack import SourceSlack from source_slack.streams import Channels, JoinChannelsStream, Threads +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + @pytest.fixture def authenticator(token_config): @@ -36,10 +37,10 @@ def get_stream_by_name(stream_name, config): {}, [ # two messages per each channel - {'channel': 'airbyte-for-beginners', 'ts': 1577866844}, - {'channel': 'airbyte-for-beginners', 'ts': 1577877406}, - {'channel': 'good-reads', 'ts': 1577866844}, - {'channel': 'good-reads', 'ts': 1577877406}, + {"channel": "airbyte-for-beginners", "ts": 1577866844}, + {"channel": "airbyte-for-beginners", "ts": 1577877406}, + {"channel": "good-reads", "ts": 1577866844}, + {"channel": "good-reads", "ts": 1577877406}, ], ), ("2020-01-02T00:00:00Z", "2020-01-01T00:00:00Z", [], {}, [{}]), @@ -55,18 +56,18 @@ def get_stream_by_name(stream_name, config): ), ), ) -def test_threads_stream_slices( - requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result -): +def test_threads_stream_slices(requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result): token_config["channel_filter"] = [] requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) start_date = pendulum.parse(start_date) @@ -76,7 +77,7 @@ def test_threads_stream_slices( authenticator=authenticator, default_start_date=start_date, end_date=end_date, - lookback_window=pendulum.Duration(days=token_config["lookback_window"]) + lookback_window=pendulum.Duration(days=token_config["lookback_window"]), ) slices = list(stream.stream_slices(stream_state=stream_state)) assert slices == expected_result @@ -92,11 +93,10 @@ def test_threads_stream_slices( ), ) def test_get_updated_state(authenticator, token_config, current_state, latest_record, expected_state): - stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) assert stream._get_updated_state(current_stream_state=current_state, latest_record=latest_record) == expected_state @@ -105,10 +105,10 @@ def test_threads_request_params(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - expected = {'channel': 'airbyte-for-beginners', 'limit': 1000, 'ts': 1577866844} + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + expected = {"channel": "airbyte-for-beginners", "limit": 1000, "ts": 1577866844} assert stream.request_params(stream_slice=threads_slice, stream_state={}) == expected @@ -116,7 +116,7 @@ def test_threads_parse_response(mocker, authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) resp = { "messages": [ @@ -129,14 +129,14 @@ def test_threads_parse_response(mocker, authenticator, token_config): "subscribed": True, "last_read": "1484678597.521003", "unread_count": 0, - "ts": "1482960137.003543" + "ts": "1482960137.003543", } ] } resp_mock = mocker.Mock() resp_mock.json.return_value = resp - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - actual_response = list(stream.parse_response(response=resp_mock,stream_slice=threads_slice)) + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + actual_response = list(stream.parse_response(response=resp_mock, stream_slice=threads_slice)) assert len(actual_response) == 1 assert actual_response[0]["float_ts"] == 1482960137.003543 assert actual_response[0]["channel_id"] == "airbyte-for-beginners" @@ -147,7 +147,7 @@ def test_backoff(token_config, authenticator, headers, expected_result): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, headers=headers) assert stream.get_backoff_strategy().backoff_time(mocked_response) == expected_result @@ -157,10 +157,7 @@ def test_channels_stream_with_autojoin(authenticator) -> None: """ The test uses the `conversations_list` fixture(autouse=true) as API mocker. """ - expected = [ - {'id': 'airbyte-for-beginners', 'is_member': True}, - {'id': 'good-reads', 'is_member': True} - ] + expected = [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}] stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) assert list(stream.read_records(None)) == expected @@ -169,7 +166,7 @@ def test_next_page_token(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = Mock() mocked_response.json.return_value = {"response_metadata": {"next_cursor": "next page"}} @@ -189,22 +186,24 @@ def test_should_retry(authenticator, token_config, status_code, expected): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, status_code=status_code) mocked_response.ok = status_code == 200 assert stream.get_error_handler().interpret_response(mocked_response).response_action == expected + def test_channels_stream_with_include_private_channels_false(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=False, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel' + assert params.get("types") == "public_channel" + def test_channels_stream_with_include_private_channels(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=True, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel,private_channel' + assert params.get("types") == "public_channel,private_channel" diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smartengage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-smartsheets/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-smartsheets/main.py b/airbyte-integrations/connectors/source-smartsheets/main.py index 62f5650b92eac..deabf33cba73d 100644 --- a/airbyte-integrations/connectors/source-smartsheets/main.py +++ b/airbyte-integrations/connectors/source-smartsheets/main.py @@ -4,5 +4,6 @@ from source_smartsheets.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py index ac0c398907e07..f21e72409f4a3 100644 --- a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple import smartsheet + from airbyte_cdk.sources.streams.http.requests_native_auth import SingleUseRefreshTokenOauth2Authenticator diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py index 54e3bb226ee5d..285e1da0e19b0 100644 --- a/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-smartsheets/unit_tests/conftest.py @@ -9,6 +9,7 @@ import pytest from smartsheet.models import Sheet + HERE = Path(__file__).parent.absolute() diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py index 0e9fd35ce43d4..498fd9fb81928 100644 --- a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_streams.py @@ -5,9 +5,10 @@ import datetime from unittest.mock import Mock -from airbyte_cdk.models import SyncMode from source_smartsheets.streams import SmartsheetStream +from airbyte_cdk.models import SyncMode + def test_state_saved_after_each_record(config, get_sheet_mocker): today_dt = datetime.datetime.now(datetime.timezone.utc) diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-spacex-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-square/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-statuspage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index 971f33a69dd10..e81b1f8f0da51 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -5,5 +5,6 @@ from source_stripe.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py index 7365b5902b938..c909371d1ede1 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/parent_incremental_stripe_sub_stream_error_handler.py @@ -5,6 +5,7 @@ from typing import Optional, Union import requests + from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution from source_stripe.error_handlers.stripe_error_handler import StripeErrorHandler diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py index a9fc2f3e93076..315b03c9dbdee 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_handlers/stripe_error_handler.py @@ -6,11 +6,13 @@ from typing import Optional, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + STRIPE_ERROR_CODES = { "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " "Try to grant required permissions/scopes or re-authenticate", diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py b/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py index 732ac748556d4..70cb8d1d425e0 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/error_mappings/parent_incremental_stripe_sub_stream_error_mapping.py @@ -5,6 +5,7 @@ from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction + PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING = DEFAULT_ERROR_MAPPING | { 404: ErrorResolution( response_action=ResponseAction.IGNORE, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py index 37263eabb998d..e6878a153f191 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -8,9 +8,10 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_stripe import SourceStripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 5ce96c24cc825..7002a36f5a4a7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum + from airbyte_cdk.entrypoint import logger as entrypoint_logger from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource @@ -36,6 +37,7 @@ UpdatedCursorIncrementalStripeSubStream, ) + logger = logging.getLogger("airbyte") _MAX_CONCURRENCY = 20 @@ -46,7 +48,6 @@ class SourceStripe(ConcurrentSourceAdapter): - message_repository = InMemoryMessageRepository(entrypoint_logger.level) _SLICE_BOUNDARY_FIELDS_BY_IMPLEMENTATION = { Events: ("created[gte]", "created[lte]"), @@ -583,7 +584,10 @@ def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: ), StripeSubStream( name="usage_records", - path=lambda self, stream_slice, *args, **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", + path=lambda self, + stream_slice, + *args, + **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", parent=subscription_items, primary_key=None, **args, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 5eec772a4f858..3f1ca33aefea0 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -11,6 +11,7 @@ import pendulum import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies import ExponentialBackoffStrategy @@ -24,6 +25,7 @@ from source_stripe.error_handlers import ParentIncrementalStripeSubStreamErrorHandler, StripeErrorHandler from source_stripe.error_mappings import PARENT_INCREMENTAL_STRIPE_SUB_STREAM_ERROR_MAPPING + STRIPE_API_VERSION = "2022-11-15" CACHE_DISABLED = os.environ.get("CACHE_DISABLED") IS_TESTING = os.environ.get("DEPLOYMENT_MODE") == "testing" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index f0bcd0cb9e711..da3a15fd2e0fc 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -5,10 +5,12 @@ import os import pytest + from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.test.state_builder import StateBuilder + os.environ["CACHE_DISABLED"] = "true" os.environ["DEPLOYMENT_MODE"] = "testing" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py index 35c72a1938748..ce2e72a67d483 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_accounts.py @@ -4,6 +4,8 @@ from unittest import TestCase import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -20,7 +22,7 @@ from integration.config import ConfigBuilder from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder -from source_stripe import SourceStripe + _STREAM_NAME = "accounts" _ACCOUNT_ID = "acct_1G9HZLIEn49ers" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py index 9e8ca1c2628cb..dd7a36ebd56a1 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, AirbyteStateMessage, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["application_fee.created", "application_fee.refunded"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py index 1e8b500ea9886..8581688de825a 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py @@ -8,6 +8,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -29,7 +31,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["application_fee.refund.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py index cdb0da90e26a0..c4a29d2d42d6d 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py index bafb8da0bb454..92abc3a33af2b 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py @@ -8,6 +8,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -29,7 +31,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py index 660564ea96207..d1c037ea9e650 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_card.created", "issuing_card.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py index b64ba61d6db85..7e0c41a080f4c 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["radar.early_fraud_warning.created", "radar.early_fraud_warning.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py index 87c5494b755e2..4215ead25260f 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _STREAM_NAME = "events" _NOW = datetime.now(timezone.utc) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py index c9f7142af0598..95ae64e494a7d 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py index 918cab91a40d8..3e4f6c2adfa7f 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py index f41f2efbbbf5c..1c9bb3a5e2af5 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -28,7 +30,7 @@ from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status from integration.test_bank_accounts import _a_customer, _customers_response -from source_stripe import SourceStripe + _EVENT_TYPES = ["payment_method.*"] @@ -105,12 +107,7 @@ class FullRefreshTest(TestCase): def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -125,12 +122,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -152,12 +144,7 @@ def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMo def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -172,12 +159,7 @@ def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -190,12 +172,7 @@ def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mock def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -208,12 +185,7 @@ def test_given_http_status_401_when_read_then_config_error(self, http_mocker: Ht def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -229,12 +201,7 @@ def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_m def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -247,12 +214,7 @@ def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_r def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -272,12 +234,7 @@ class IncrementalTest(TestCase): def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) cursor_value = int(_A_START_DATE.timestamp()) + 1 http_mocker.get( diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py index f1a96ed48693e..58a3b355ebd61 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payout_balance_transactions.py @@ -4,6 +4,8 @@ from unittest import TestCase import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read @@ -21,7 +23,7 @@ from integration.config import ConfigBuilder from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder -from source_stripe import SourceStripe + _STREAM_NAME = "payout_balance_transactions" _A_PAYOUT_ID = "a_payout_id" @@ -118,7 +120,10 @@ def test_given_multiple_parents_when_read_then_extract_from_all_children(self, h config = _config().with_start_date(_START_DATE).build() http_mocker.get( _payouts_request().with_created_gte(_START_DATE).with_created_lte(_NOW).with_limit(100).build(), - _payouts_response().with_record(_create_payout_record().with_id(_A_PAYOUT_ID)).with_record(_create_payout_record().with_id(_ANOTHER_PAYOUT_ID)).build(), + _payouts_response() + .with_record(_create_payout_record().with_id(_A_PAYOUT_ID)) + .with_record(_create_payout_record().with_id(_ANOTHER_PAYOUT_ID)) + .build(), ) http_mocker.get( _balance_transactions_request().with_limit(100).with_payout(_A_PAYOUT_ID).build(), @@ -160,8 +165,15 @@ def test_when_read_then_fetch_from_updated_payouts(self, http_mocker: HttpMocker state = StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(_STATE_DATE.timestamp())}).build() catalog = _create_catalog(SyncMode.incremental) http_mocker.get( - _events_request().with_created_gte(_STATE_DATE + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), - _events_response().with_record(_event_record().with_field(_DATA_FIELD, _create_payout_record().with_id(_A_PAYOUT_ID).build())).build(), + _events_request() + .with_created_gte(_STATE_DATE + _AVOIDING_INCLUSIVE_BOUNDARIES) + .with_created_lte(_NOW) + .with_limit(100) + .with_types(_EVENT_TYPES) + .build(), + _events_response() + .with_record(_event_record().with_field(_DATA_FIELD, _create_payout_record().with_id(_A_PAYOUT_ID).build())) + .build(), ) http_mocker.get( _balance_transactions_request().with_limit(100).with_payout(_A_PAYOUT_ID).build(), diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py index 7779b39bcd03f..de4dd3d3126bf 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, AirbyteStreamStatus, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -26,7 +28,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _STREAM_NAME = "persons" _ACCOUNT_ID = "acct_1G9HZLIEn49ers" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py index 13da98982181c..7bc0c7f3d3296 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["review.closed", "review.opened"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py index cf5b850124d0b..280df52183524 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py @@ -6,6 +6,8 @@ from unittest.mock import patch import freezegun +from source_stripe import SourceStripe + from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, FailureType, StreamDescriptor, SyncMode from airbyte_cdk.sources.source import TState from airbyte_cdk.sources.streams.http.error_handlers.http_status_error_handler import HttpStatusErrorHandler @@ -27,7 +29,7 @@ from integration.pagination import StripePaginationStrategy from integration.request_builder import StripeRequestBuilder from integration.response_builder import a_response_with_status -from source_stripe import SourceStripe + _EVENT_TYPES = ["issuing_transaction.created", "issuing_transaction.updated"] diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 1567a930ebbee..98128124534e0 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -6,12 +6,14 @@ from contextlib import nullcontext as does_not_raise import pytest +from source_stripe import SourceStripe + from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer, SyncMode from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.test.state_builder import StateBuilder from airbyte_cdk.utils import AirbyteTracedException -from source_stripe import SourceStripe + logger = logging.getLogger("airbyte") _ANY_CATALOG = ConfiguredAirbyteCatalogSerializer.load({"streams": []}) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index eaaafcad12ef3..666dcb9e961fd 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -133,7 +133,6 @@ def test_lazy_substream_data_cursor_value_is_populated( def test_lazy_substream_data_is_expanded( requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state ): - config["start_date"] = str(pendulum.today().subtract(days=3)) stream = stream_by_name("bank_accounts", config) for url, body in requests_mock_map.items(): @@ -523,6 +522,7 @@ def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by ] assert records == [{"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1691629292}] + @freezegun.freeze_time("2023-08-23T15:00:15Z") def test_setup_attempts(requests_mock, incremental_stream_args): requests_mock.get( @@ -583,12 +583,12 @@ def test_setup_attempts(requests_mock, incremental_stream_args): def test_persons_wo_state(requests_mock, stream_args): requests_mock.get("/v1/accounts", json={"data": [{"id": 1, "object": "account", "created": 111}]}) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("full_refresh")) assert slices == [{"parent": {"id": 1, "object": "account", "created": 111}}] requests_mock.get("/v1/accounts/1/persons", json={"data": [{"id": 11, "object": "person", "created": 222}]}) @@ -618,12 +618,12 @@ def test_persons_w_state(requests_mock, stream_args): }, ) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("incremental", stream_state={"updated": pendulum.parse("2023-08-20T00:00:00").int_timestamp})) assert slices == [{}] records = [ @@ -802,7 +802,7 @@ def test_subscription_items_extra_request_params(requests_mock, stream_by_name, "created": 1699603175, "quantity": 1, "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", - "subscription_updated": 1699603174, #1699603175 + "subscription_updated": 1699603174, # 1699603175 }, { "id": "si_OynPdzMZykmCWm", diff --git a/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-surveycto/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveycto/main.py b/airbyte-integrations/connectors/source-surveycto/main.py index 9f282dbc2ecd4..7bd10098aad5f 100644 --- a/airbyte-integrations/connectors/source-surveycto/main.py +++ b/airbyte-integrations/connectors/source-surveycto/main.py @@ -4,5 +4,6 @@ from source_surveycto.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py index 816b0dad755eb..f0ff2adbb409f 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream @@ -111,7 +112,6 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: # Source class SourceSurveycto(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, Any]: - form_ids = config["form_id"] try: diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 77ebfa14f6246..863aa11f2b76e 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -26,9 +26,11 @@ def source_fixture(): @pytest.fixture(name="mock_survey_cto") def mock_survey_cto_fixture(): - with patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, patch( - "source_surveycto.source.Helpers.get_filter_data", return_value="value" - ) as mock_filter_data, patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema: + with ( + patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, + patch("source_surveycto.source.Helpers.get_filter_data", return_value="value") as mock_filter_data, + patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema, + ): yield mock_call_survey_cto, mock_filter_data, mock_json_schema diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-surveymonkey/main.py b/airbyte-integrations/connectors/source-surveymonkey/main.py index bf4f900ad3776..4daa260e708e9 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/main.py +++ b/airbyte-integrations/connectors/source-surveymonkey/main.py @@ -4,5 +4,6 @@ from source_surveymonkey.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py index 98709c3e5f930..8c96e04e6453b 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/config_migrations.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + logger = logging.getLogger("airbyte_logger") diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py index 18dcacd06ea20..f2951a5f1903c 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/source.py @@ -7,12 +7,14 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from .streams import Surveys + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py index 33f1740a68e45..ff46c119e1efb 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/source_surveymonkey/streams.py @@ -10,11 +10,13 @@ import pendulum import requests import vcr + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import CheckpointMixin from airbyte_cdk.sources.streams.http import HttpStream + cache_file = tempfile.NamedTemporaryFile() diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py index 74c9ae98ad013..0d1451270f991 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py @@ -6,37 +6,37 @@ import pendulum import pytest +from source_surveymonkey.source import SourceSurveymonkey + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import StreamSlice -from source_surveymonkey.source import SourceSurveymonkey -@pytest.fixture(name='read_json') +@pytest.fixture(name="read_json") def read_json_fixture(request): def read_json(file_name, skip_folder=False): if not skip_folder: - folder_name = request.node.fspath.basename.split('.')[0] + folder_name = request.node.fspath.basename.split(".")[0] with open("unit_tests/" + folder_name + "/" + file_name) as f: return json.load(f) + return read_json -@pytest.fixture(name='read_records') + +@pytest.fixture(name="read_records") def read_records_fixture(config): def read_records(stream_name, slice=StreamSlice(partition={"survey_id": "307785415"}, cursor_slice={})): stream = next(filter(lambda x: x.name == stream_name, SourceSurveymonkey().streams(config=config))) - records = list( - map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) + records = list(map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) return records + return read_records @pytest.fixture def args_mock(): - return { - "authenticator": None, - "start_date": pendulum.parse("2000-01-01"), - "survey_ids": [] - } + return {"authenticator": None, "start_date": pendulum.parse("2000-01-01"), "survey_ids": []} + @pytest.fixture def config(args_mock): @@ -44,5 +44,5 @@ def config(args_mock): **args_mock, "survey_ids": ["307785415"], "credentials": {"access_token": "access_token"}, - "start_date": args_mock["start_date"].to_iso8601_string() + "start_date": args_mock["start_date"].to_iso8601_string(), } diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py index 4e4fde8edf321..8f42734f7fd00 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_config_migrations.py @@ -8,6 +8,7 @@ from source_surveymonkey.config_migrations import MigrateAccessTokenToCredentials from source_surveymonkey.source import SourceSurveymonkey + TEST_CONFIG = "test_old_config.json" NEW_TEST_CONFIG = "test_new_config.json" UPGRADED_TEST_CONFIG = "test_upgraded_config.json" diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py index eaa7b4d151767..5af6707b9a7b6 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_custom_router.py @@ -6,9 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from source_surveymonkey.components import SurveyIdPartitionRouter +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig + + # test cases as a list of tuples (survey_ids, parent_stream_configs, expected_slices) test_cases = [ ( diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py index 32ef91a401c61..9800cc794fd71 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_for_updated_state.py @@ -4,9 +4,10 @@ import pendulum import pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_surveymonkey.streams import Surveys +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + class TestSurveymonkeySource: @staticmethod diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py index a397163a108bf..69850063503bf 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py @@ -4,6 +4,7 @@ from source_surveymonkey.source import SourceSurveymonkey + source_config = {"start_date": "2021-01-01T00:00:00", "access_token": "something"} new_source_config = { "start_date": "2021-01-01T00:00:00", diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py index 24a60bc08d959..66a14003d8998 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -5,14 +5,15 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import SyncMode from source_surveymonkey.streams import SurveyIds, Surveys +from airbyte_cdk.models import SyncMode + -@pytest.mark.parametrize("stream, expected_records_file, stream_slice", [ - (SurveyIds, "records_survey_ids.json", None), - (Surveys, "records_surveys.json", {"survey_id": "307785415"}) -]) +@pytest.mark.parametrize( + "stream, expected_records_file, stream_slice", + [(SurveyIds, "records_survey_ids.json", None), (Surveys, "records_surveys.json", {"survey_id": "307785415"})], +) def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, expected_records_file, stream_slice): requests_mock.get( "https://api.surveymonkey.com/v3/surveys", @@ -30,8 +31,11 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, } }, }, - {"status_code": 200, "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, - "json": read_json("response_survey_ids.json")}, + { + "status_code": 200, + "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, + "json": read_json("response_survey_ids.json"), + }, ], ) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) @@ -42,10 +46,10 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, assert list(records) == expected_records -@pytest.mark.parametrize("additional_arguments, expected_slices", [ - ({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), - ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}]) -]) +@pytest.mark.parametrize( + "additional_arguments, expected_slices", + [({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}])], +) def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments, expected_slices): if not additional_arguments: requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=read_json("response_survey_ids.json")) @@ -54,11 +58,14 @@ def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments assert list(stream_slices) == expected_slices -@pytest.mark.parametrize("endpoint, records_filename", [ - ("survey_pages", "records_survey_pages.json"), - ("survey_questions", "records_survey_questions.json"), - ("survey_collectors", "records_survey_collectors.json") -]) +@pytest.mark.parametrize( + "endpoint, records_filename", + [ + ("survey_pages", "records_survey_pages.json"), + ("survey_questions", "records_survey_questions.json"), + ("survey_collectors", "records_survey_collectors.json"), + ], +) def test_survey_data(requests_mock, read_records, read_json, endpoint, records_filename): requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/collectors", json=read_json("response_survey_collectors.json")) diff --git a/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tempo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-teradata/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/components.py b/airbyte-integrations/connectors/source-the-guardian-api/components.py index 9981873147808..8d54333154f20 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/components.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/components.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional import requests + from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement diff --git a/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py index 235d12e640af0..1e2ca8425a3f1 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py @@ -9,60 +9,57 @@ def create_response(current_page: int, total_pages: int) -> requests.Response: """Helper function to create mock responses""" response = MagicMock(spec=requests.Response) - response.json.return_value = { - "response": { - "currentPage": current_page, - "pages": total_pages - } - } + response.json.return_value = {"response": {"currentPage": current_page, "pages": total_pages}} return response + @pytest.mark.parametrize( "current_page,total_pages,expected_next_page", [ - (1, 5, 2), # First page - (2, 5, 3), # Middle page - (4, 5, 5), # Second to last page - (5, 5, None), # Last page - (1, 1, None), # Single page + (1, 5, 2), # First page + (2, 5, 3), # Middle page + (4, 5, 5), # Second to last page + (5, 5, None), # Last page + (1, 1, None), # Single page ], - ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"] + ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"], ) def test_page_increment(connector_dir, components_module, current_page, total_pages, expected_next_page): """Test the CustomPageIncrement pagination for various page combinations""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Set internal page counter to match current_page paginator._page = current_page - + mock_response = create_response(current_page, total_pages) next_page = paginator.next_page_token(mock_response) assert next_page == expected_next_page, f"Page {current_page} of {total_pages} should get next_page={expected_next_page}" + def test_reset_functionality(components_module): """Test the reset behavior of CustomPageIncrement""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Advance a few pages mock_response = create_response(current_page=1, total_pages=5) paginator.next_page_token(mock_response) paginator.next_page_token(create_response(current_page=2, total_pages=5)) - + # Test reset paginator.reset() assert paginator._page == 1, "Reset should set page back to 1" - + # Verify pagination works after reset next_page = paginator.next_page_token(mock_response) assert next_page == 2, "Should increment to page 2 after reset" diff --git a/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tidb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/main.py b/airbyte-integrations/connectors/source-tiktok-marketing/main.py index b523ea1b0fdd2..cc7653e249d5f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/main.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/main.py @@ -4,5 +4,6 @@ from source_tiktok_marketing.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py index f74710d7d2ff3..d7ad3129553a3 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/advertiser_ids_partition_router.py @@ -4,6 +4,7 @@ from typing import Any, Iterable, Mapping import dpath.util + from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter from airbyte_cdk.sources.declarative.types import StreamSlice diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py index 628a57a7a89fe..21b5ae6ecceaa 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py @@ -7,7 +7,6 @@ class PerPartitionRecordFilter(RecordFilter): - """ Prepares per partition stream state to be used in the Record Filter condition. Gets current stream state cursor value for stream slice and passes it to condition. diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py index 0b6883e6d13d0..ece17f12fc701 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/transformations.py @@ -18,7 +18,6 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Mapping[str, Any]: - for metric_key, metric_value in record.get("metrics", {}).items(): if metric_value == self.empty_value: record["metrics"][metric_key] = None diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 8d0b1f1afcc24..6d2655f5ccfba 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream + logger = logging.getLogger("airbyte") DOCUMENTATION_URL = "https://docs.airbyte.com/integrations/sources/tiktok-marketing" SANDBOX_STREAM_NAMES = [ diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py index 9a0abf1bc2538..dba9ba324d682 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/advetiser_slices.py @@ -5,6 +5,7 @@ from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template + ADVERTISERS_FILE = "advertisers" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py index 5c32c0b9ea7d8..c01481cb119e5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py @@ -7,14 +7,9 @@ class ConfigBuilder: def __init__(self) -> None: self._config: Dict[str, Any] = { - "credentials": { - "auth_type": "oauth2.0", - "access_token": "access token", - "app_id": "11111111111111111111", - "secret": "secret" - }, + "credentials": {"auth_type": "oauth2.0", "access_token": "access token", "app_id": "11111111111111111111", "secret": "secret"}, "start_date": "2024-01-01", - "include_deleted": False + "include_deleted": False, } def with_include_deleted(self) -> "ConfigBuilder": diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py index f3f40d4f3e6ef..501f144efcc7a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_music.py @@ -4,13 +4,14 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing class TestCreativeAssetsMusic(TestCase): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py index 19a875a648eff..10dd031fecf47 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_creative_assets_portfolios.py @@ -4,13 +4,14 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing class TestCreativeAssetsPortfolios(TestCase): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py index b0a8f102ce3dc..0f72549ef59c2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py @@ -4,14 +4,16 @@ from unittest import TestCase from advetiser_slices import mock_advertisers_slices +from config_builder import ConfigBuilder +from source_tiktok_marketing import SourceTiktokMarketing + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse from airbyte_cdk.test.mock_http.response_builder import find_template from airbyte_cdk.test.state_builder import StateBuilder -from config_builder import ConfigBuilder -from source_tiktok_marketing import SourceTiktokMarketing + EMPTY_LIST_RESPONSE = {"code": 0, "message": "ok", "data": {"list": []}} @@ -88,10 +90,11 @@ class TestAdsReportHourly(TestCase): "real_time_app_install_cost", "app_install", ] + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -113,16 +116,16 @@ def state(self): def mock_response(self, http_mocker: HttpMocker, include_deleted=False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_AD", - "dimensions": '["ad_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_AD", + "dimensions": '["ad_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "ad_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( @@ -132,7 +135,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted=False): ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), ) - query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] + query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( @@ -239,7 +242,7 @@ class TestAdGroupsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -263,16 +266,16 @@ def state(self): def test_basic_read(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -348,17 +351,17 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) filtering = '[{"field_name": "adgroup_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - "filtering": filtering, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + "filtering": filtering, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -380,6 +383,7 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): assert output.records[0].record.data.get("adgroup_id") is not None assert output.records[0].record.data.get("stat_time_hour") is not None + class TestAdvertisersReportsHourly(TestCase): stream_name = "advertisers_reports_hourly" advertiser_id = "872746382648" @@ -536,7 +540,7 @@ class TestCampaignsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -556,18 +560,18 @@ def state(self): .build() ) - def mock_response(self, http_mocker: HttpMocker, include_deleted:bool=False): + def mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_CAMPAIGN", - "dimensions": '["campaign_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_CAMPAIGN", + "dimensions": '["campaign_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "campaign_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py index 1825fa49d955b..ce1e6432c5764 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py @@ -3,9 +3,6 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.declarative.types import StreamSlice from source_tiktok_marketing import SourceTiktokMarketing from source_tiktok_marketing.components.advertiser_ids_partition_router import ( MultipleAdvertiserIdsPerPartition, @@ -15,13 +12,17 @@ from source_tiktok_marketing.components.semi_incremental_record_filter import PerPartitionRecordFilter from source_tiktok_marketing.components.transformations import TransformEmptyMetrics +from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.declarative.types import StreamSlice + @pytest.mark.parametrize( "config, expected", [ ({"credentials": {"advertiser_id": "11111111111"}}, "11111111111"), ({"environment": {"advertiser_id": "2222222222"}}, "2222222222"), - ({"credentials": {"access_token": "access_token"}}, None) + ({"credentials": {"access_token": "access_token"}}, None), ], ) def test_get_partition_value_from_config(config, expected): @@ -29,8 +30,9 @@ def test_get_partition_value_from_config(config, expected): parent_stream_configs=[MagicMock()], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) actual = router.get_partition_value_from_config() assert actual == expected @@ -39,17 +41,26 @@ def test_get_partition_value_from_config(config, expected): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, - [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_ids": '["2222222222"]', "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], - {"code": 0, "message": "ok", "data": - {"list": [{"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_multiple(config, expected, requests_mock, json_data): @@ -57,23 +68,19 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = MultipleAdvertiserIdsPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_ids", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_ids", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_ids" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_ids", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -81,19 +88,26 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, [{"advertiser_id": "11111111111", "parent_slice": {}}], - None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_id": "11111111111", "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_id": "2222222222", "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_id": "11111111", "parent_slice": {}}, - {"advertiser_id": "22222222", "parent_slice": {}}], - {"code": 0, "message": "ok", - "data": {"list": [ - {"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_id": "11111111", "parent_slice": {}}, {"advertiser_id": "22222222", "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_single(config, expected, requests_mock, json_data): @@ -101,23 +115,19 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = SingleAdvertiserIdPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_id", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_id", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -126,22 +136,22 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): "records, state, slice, expected", [ ( - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], - {}, - {}, - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}] + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], + {}, + {}, + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], ), ( - [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 1}, - [{"advertiser_id": 1, "start_time": "2024-01-02"}] + [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 1}, + [{"advertiser_id": 1, "start_time": "2024-01-02"}], ), ( - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 2}, - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 2}, + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], ), ], ) @@ -150,18 +160,20 @@ def test_record_filter(records, state, slice, expected): record_filter = PerPartitionRecordFilter( config=config, parameters={"partition_field": "advertiser_id"}, - condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}" + condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}", + ) + filtered_records = list( + record_filter.filter_records(records=records, stream_state=state, stream_slice=StreamSlice(partition=slice, cursor_slice={})) ) - filtered_records = list(record_filter.filter_records( - records=records, - stream_state=state, - stream_slice=StreamSlice(partition=slice, cursor_slice={}) - )) assert filtered_records == expected def test_hourly_datetime_based_cursor(): - config = {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, "start_date": "2022-01-01", "end_date": "2022-01-02"} + config = { + "credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, + "start_date": "2022-01-01", + "end_date": "2022-01-02", + } cursor = HourlyDatetimeBasedCursor( start_datetime=MinMaxDatetime(datetime="{{ config.get('start_date', '2016-09-01') }}", datetime_format="%Y-%m-%d", parameters={}), @@ -172,42 +184,33 @@ def test_hourly_datetime_based_cursor(): cursor_field="stat_time_hour", datetime_format="%Y-%m-%d", cursor_datetime_formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"], - parameters={} + parameters={}, ) cursor._cursor = "2022-01-01 00:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] cursor._cursor = "2022-01-01 10:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] @pytest.mark.parametrize( "record, expected", [ + ({"metrics": {"metric_1": "not empty", "metric_2": "-"}}, {"metrics": {"metric_1": "not empty", "metric_2": None}}), + ({"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}), ( - {"metrics": {"metric_1": "not empty", "metric_2": "-"}}, - {"metrics": {"metric_1": "not empty", "metric_2": None}} - ), - ( - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}} - ), - ( - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}} - ), - ( - {}, - {} + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, ), + ({}, {}), ], ) def test_transform_empty_metrics(record, expected): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py index 44db480d1000d..f04efc33e7c19 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py @@ -6,17 +6,34 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import ConnectorSpecification from source_tiktok_marketing import SourceTiktokMarketing +from airbyte_cdk.models import ConnectorSpecification + @pytest.mark.parametrize( "config, stream_len", [ ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01"}, 36), ({"access_token": "token", "start_date": "2021-01-01", "environment": {"advertiser_id": "1111"}}, 28), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "LIFETIME"}, 15), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "DAY"}, 27), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "LIFETIME", + }, + 15, + ), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "DAY", + }, + 27, + ), ], ) def test_source_streams(config, stream_len): @@ -43,11 +60,27 @@ def config_fixture(): def test_source_check_connection_ok(config, requests_mock): requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/advertiser/info/?page_size=100&advertiser_ids=%5B%22917429327%22%5D", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) logger_mock = MagicMock() assert SourceTiktokMarketing().check_connection(logger_mock, config) == (True, None) @@ -56,20 +89,17 @@ def test_source_check_connection_ok(config, requests_mock): @pytest.mark.parametrize( "json_response, expected_result, expected_message", [ - ({"code": 40105, "message": "Access token is incorrect or has been revoked."}, - (False, "Access token is incorrect or has been revoked."), - None), - ({"code": 40100, "message": "App reaches the QPS limit."}, - None, - 38) - ] + ( + {"code": 40105, "message": "Access token is incorrect or has been revoked."}, + (False, "Access token is incorrect or has been revoked."), + None, + ), + ({"code": 40100, "message": "App reaches the QPS limit."}, None, 38), + ], ) @pytest.mark.usefixtures("mock_sleep") def test_source_check_connection_failed(config, requests_mock, capsys, json_response, expected_result, expected_message): - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_response - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_response) logger_mock = MagicMock() result = SourceTiktokMarketing().check_connection(logger_mock, config) diff --git a/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-timely/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tmdb/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-toggl/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tplcentral/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tplcentral/main.py b/airbyte-integrations/connectors/source-tplcentral/main.py index c5e7b8a95ec4b..fb91c7bd8e63a 100644 --- a/airbyte-integrations/connectors/source-tplcentral/main.py +++ b/airbyte-integrations/connectors/source-tplcentral/main.py @@ -4,5 +4,6 @@ from source_tplcentral.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py index a20a1c20c81d1..40bd46bfe85f0 100644 --- a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/source.py @@ -7,10 +7,11 @@ from typing import Any, List, Mapping, MutableMapping, Tuple import requests +from requests.auth import HTTPBasicAuth + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from requests.auth import HTTPBasicAuth from source_tplcentral.streams import Customers, Inventory, Items, Orders, StockDetails, StockSummaries diff --git a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py index 97e696a6da7fd..15a328c51c51f 100644 --- a/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py +++ b/airbyte-integrations/connectors/source-tplcentral/source_tplcentral/streams.py @@ -8,6 +8,7 @@ import arrow import requests + from airbyte_cdk.sources.streams.http import HttpStream from source_tplcentral.util import deep_get, normalize diff --git a/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py index b31950e754047..2a116bd8e350d 100644 --- a/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-tplcentral/unit_tests/test_incremental_streams.py @@ -3,9 +3,10 @@ # import pytest -from airbyte_cdk.models import SyncMode from source_tplcentral.streams import IncrementalTplcentralStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def config(): diff --git a/airbyte-integrations/connectors/source-trello/components.py b/airbyte-integrations/connectors/source-trello/components.py index dfcb6799f92b7..31c67c6ad62ed 100644 --- a/airbyte-integrations/connectors/source-trello/components.py +++ b/airbyte-integrations/connectors/source-trello/components.py @@ -14,7 +14,6 @@ @dataclass class OrderIdsPartitionRouter(SubstreamPartitionRouter): def stream_slices(self) -> Iterable[StreamSlice]: - stream_map = {stream_config.stream.name: stream_config.stream for stream_config in self.parent_stream_configs} board_ids = set(self.config.get("board_ids", [])) diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py index 439e5148ef2b3..6cc39f099c5de 100644 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py +++ b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py @@ -4,9 +4,10 @@ import pytest -from airbyte_cdk.sources.streams.core import Stream from source_trello.components import OrderIdsPartitionRouter +from airbyte_cdk.sources.streams.core import Stream + class MockStream(Stream): def __init__(self, records): diff --git a/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-trustpilot/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-twilio/main.py b/airbyte-integrations/connectors/source-twilio/main.py index 0999d1e67f264..972b1826fb0ac 100644 --- a/airbyte-integrations/connectors/source-twilio/main.py +++ b/airbyte-integrations/connectors/source-twilio/main.py @@ -4,5 +4,6 @@ from source_twilio.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 26c2269b16afb..b2bccf6832de3 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Tuple import pendulum + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -48,6 +49,7 @@ VerifyServices, ) + RETENTION_WINDOW_LIMIT = 400 diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 2e82d66394524..415fb33589178 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -10,13 +10,15 @@ import pendulum import requests +from pendulum.datetime import DateTime +from requests.auth import AuthBase + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum.datetime import DateTime -from requests.auth import AuthBase + TWILIO_CHAT_BASE = "https://chat.twilio.com/v2/" TWILIO_CONVERSATION_BASE = "https://conversations.twilio.com/v1/" diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py index 309355222b869..4c365be55842d 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py @@ -8,7 +8,6 @@ import pendulum import pytest import requests -from airbyte_cdk.sources.streams.http import HttpStream from freezegun import freeze_time from source_twilio.auth import HttpBasicAuthenticator from source_twilio.source import SourceTwilio @@ -27,6 +26,9 @@ UsageTriggers, ) +from airbyte_cdk.sources.streams.http import HttpStream + + TEST_CONFIG = { "account_sid": "airbyte.io", "auth_token": "secret", @@ -43,7 +45,6 @@ class TestTwilioStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( @@ -60,14 +61,14 @@ def test_data_field(self, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, expected", [ - (Accounts, ['name']), + (Accounts, ["name"]), ], ) def test_changeable_fields(self, stream_cls, expected): - with patch.object(Accounts, "changeable_fields", ['name']): - stream = stream_cls(**self.CONFIG) - result = stream.changeable_fields - assert result == expected + with patch.object(Accounts, "changeable_fields", ["name"]): + stream = stream_cls(**self.CONFIG) + result = stream.changeable_fields + assert result == expected @pytest.mark.parametrize( "stream_cls, expected", @@ -108,12 +109,12 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): with patch.object(TwilioStream, "changeable_fields", ["name"]): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = list(stream.parse_response(response)) - assert result[0]['id'] == expected[0]['id'] + stream = stream_cls(**self.CONFIG) + url = f"{stream.url_base}{stream.path()}" + requests_mock.get(url, json=test_response) + response = requests.get(url) + result = list(stream.parse_response(response)) + assert result[0]["id"] == expected[0]["id"] @pytest.mark.parametrize( "stream_cls, expected", @@ -151,14 +152,13 @@ def test_request_params(self, stream_cls, next_page_token, expected): ("Fri, 11 Dec 2020 04:28:40 +0000", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("2020-12-11T04:28:40Z", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("some_string", {}, "some_string"), - ] + ], ) def test_transform_function(self, original_value, field_schema, expected_value): assert Accounts.custom_transform_function(original_value, field_schema) == expected_value class TestIncrementalTwilioStream: - CONFIG = TEST_CONFIG CONFIG.pop("account_sid") CONFIG.pop("auth_token") @@ -256,7 +256,6 @@ def test_generate_dt_ranges(self, stream_cls, state, expected_dt_ranges): class TestTwilioNestedStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( @@ -299,7 +298,6 @@ def test_stream_slices(self, stream_cls, parent_stream, record, expected): class TestUsageNestedStream: - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-twitter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-typeform/main.py b/airbyte-integrations/connectors/source-typeform/main.py index 126dc556ff7db..a28ee8adc4c0c 100644 --- a/airbyte-integrations/connectors/source-typeform/main.py +++ b/airbyte-integrations/connectors/source-typeform/main.py @@ -4,5 +4,6 @@ from source_typeform.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py index 2ebf804b49403..c5698a824168d 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_typeform import SourceTypeform +from airbyte_cdk.entrypoint import launch + def run(): source = SourceTypeform() diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py index 58ec8391d7d76..8bcf7525d561f 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py index a841b1d266cff..ee22158b1c58b 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py @@ -1,13 +1,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from source_typeform.components import TypeformAuthenticator + from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator -from source_typeform.components import TypeformAuthenticator def test_typeform_authenticator(): config = {"credentials": {"auth_type": "access_token", "access_token": "access_token"}} - oauth_config = {"credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} + oauth_config = { + "credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"} + } class TokenProvider: def get_token(self) -> str: @@ -16,13 +19,13 @@ def get_token(self) -> str: auth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(auth, BearerAuthenticator) oauth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=oauth_config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(oauth, DeclarativeSingleUseRefreshTokenOauth2Authenticator) diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py index c5c3f1508e76b..f8f11245c5c38 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py @@ -6,9 +6,11 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from source_typeform.components import FormIdPartitionRouter, TypeformAuthenticator +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig + + # test cases as a list of tuples (form_ids, parent_stream_configs, expected_slices) test_cases = [ ( diff --git a/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-unleash/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-us-census/components.py b/airbyte-integrations/connectors/source-us-census/components.py index 431d0794f7ad7..73087fa6c63a1 100644 --- a/airbyte-integrations/connectors/source-us-census/components.py +++ b/airbyte-integrations/connectors/source-us-census/components.py @@ -7,6 +7,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.requesters.error_handlers import DefaultErrorHandler @@ -116,7 +117,6 @@ class USCensusErrorHandler(DefaultErrorHandler): """ def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]]) -> ErrorResolution: - if self.response_filters: for response_filter in self.response_filters: matched_error_resolution = response_filter.matches(response_or_exception=response_or_exception) diff --git a/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-us-census/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-vantage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-vitally/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-waiteraid/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-weatherstack/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-webflow/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-webflow/main.py b/airbyte-integrations/connectors/source-webflow/main.py index 4d481e07151bf..5f67a19aa9a73 100644 --- a/airbyte-integrations/connectors/source-webflow/main.py +++ b/airbyte-integrations/connectors/source-webflow/main.py @@ -4,5 +4,6 @@ from source_webflow.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py index f7e3daa64937d..4eacbe1bfcd8e 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -15,6 +16,7 @@ from .auth import WebflowTokenAuthenticator from .webflow_to_airbyte_mapping import WebflowToAirbyteMapping + """ This module is used for pulling the contents of "collections" out of Webflow, which is a CMS for hosting websites. A Webflow collection may be a group of items such as "Blog Posts", "Blog Authors", etc. @@ -201,7 +203,6 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - # Webflow default pagination is 100, for debugging pagination we set this to a low value. # This should be set back to 100 for production params = {"limit": 100} @@ -256,7 +257,6 @@ def get_json_schema(self) -> Mapping[str, Any]: class SourceWebflow(AbstractSource): - """This is the main class that defines the methods that will be called by Airbyte infrastructure""" @staticmethod diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py index ea40dc0ab320c..4b60619846493 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py @@ -4,7 +4,6 @@ class WebflowToAirbyteMapping: - """ The following disctionary is used for dynamically pulling the schema from Webflow, and mapping it to an Airbyte-compatible json-schema Webflow: https://developers.webflow.com/#get-collection-with-full-schema diff --git a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-woocommerce/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py index aaeb7f6c2529d..a56a495fcd927 100644 --- a/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-workramp/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-xero/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-xero/main.py b/airbyte-integrations/connectors/source-xero/main.py index d765f10d2093b..40c7193738367 100644 --- a/airbyte-integrations/connectors/source-xero/main.py +++ b/airbyte-integrations/connectors/source-xero/main.py @@ -4,5 +4,6 @@ from source_xero.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-xero/source_xero/components.py b/airbyte-integrations/connectors/source-xero/source_xero/components.py index 2294372e6ab55..a9e4a6655cd36 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/components.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/components.py @@ -9,6 +9,7 @@ import dpath.util import requests + from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-xero/source_xero/run.py b/airbyte-integrations/connectors/source-xero/source_xero/run.py index fb8d5955af03d..99bb37a63595f 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/run.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_xero import SourceXero +from airbyte_cdk.entrypoint import launch + def run(): source = SourceXero() diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index 3209646cb044f..7da885250cc58 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py index a2dcb6a4541ac..1f8b9a330d540 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py @@ -4,10 +4,11 @@ from typing import Any, Mapping -from airbyte_cdk.sources.streams import Stream from pytest import fixture from source_xero.source import SourceXero +from airbyte_cdk.sources.streams import Stream + @fixture(name="config_pass") def config_fixture(): diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py index 13e3b37c9f128..17795ec5716f3 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py @@ -4,13 +4,16 @@ import datetime -from airbyte_cdk.models import SyncMode from conftest import get_stream_by_name from source_xero.components import ParseDates +from airbyte_cdk.models import SyncMode + def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -22,7 +25,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py index ee4687b936ef2..3765b99482f84 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py @@ -22,6 +22,7 @@ def test_check_connection_failed(bad_config, requests_mock): assert check_succeeded is False assert error == "" or "none" in error.lower() + def test_streams_count(config_pass): source = SourceXero() streams = source.streams(config_pass) diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py index a0bdf0e116109..6dffe0aeb3d29 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py @@ -4,13 +4,16 @@ import datetime -from airbyte_cdk.models import SyncMode from conftest import get_stream_by_name from source_xero.components import ParseDates +from airbyte_cdk.models import SyncMode + def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -26,8 +29,8 @@ def test_request_params(config_pass): def test_request_headers(config_pass): bank_transactions = get_stream_by_name("bank_transactions", config_pass) - expected_headers = {'Xero-Tenant-Id': 'goodone', 'Accept': 'application/json'} - assert bank_transactions.retriever.requester.get_request_headers() == expected_headers + expected_headers = {"Xero-Tenant-Id": "goodone", "Accept": "application/json"} + assert bank_transactions.retriever.requester.get_request_headers() == expected_headers def test_http_method(config_pass): @@ -38,7 +41,7 @@ def test_http_method(config_pass): def test_ignore_forbidden(requests_mock, config_pass): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{ "message": "Forbidden resource"}]) + requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{"message": "Forbidden resource"}]) stream = get_stream_by_name("bank_transactions", config_pass) records = [] @@ -52,7 +55,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/main.py b/airbyte-integrations/connectors/source-yandex-metrica/main.py index a84b23e0a261d..ad216843240ae 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/main.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/main.py @@ -4,5 +4,6 @@ from source_yandex_metrica.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py index 874910073f0eb..e917e2fa28ed9 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/source.py @@ -6,11 +6,13 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from .streams import Sessions, Views, YandexMetricaStream + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py index 957030f23f9dc..65592bdb71540 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/source_yandex_metrica/streams.py @@ -12,11 +12,13 @@ import pendulum import requests +from pendulum import DateTime + from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams.core import IncrementalMixin, StreamData from airbyte_cdk.sources.streams.http import HttpStream -from pendulum import DateTime + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py index 3993932b1c693..bf5adface5a23 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py @@ -8,6 +8,7 @@ import pytest from source_yandex_metrica.source import SourceYandexMetrica + logger = logging.getLogger("test_source") diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py index afd233b117908..cec63d87d2d29 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py @@ -3,9 +3,11 @@ # -from airbyte_cdk.models import SyncMode from source_yandex_metrica.streams import Sessions +from airbyte_cdk.models import SyncMode + + EXPECTED_RECORDS = [ {"watchID": "00000000", "dateTime": "2022-09-01T12:00:00+00:00"}, {"watchID": "00000001", "dateTime": "2022-08-01T12:00:10+00:00"}, diff --git a/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-yotpo/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-younium/components.py b/airbyte-integrations/connectors/source-younium/components.py index 7b9bcaa42a5fb..f202ba2a73ec4 100644 --- a/airbyte-integrations/connectors/source-younium/components.py +++ b/airbyte-integrations/connectors/source-younium/components.py @@ -7,10 +7,12 @@ from typing import Any, Mapping, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response # The Bearer token generated by server-to-server token will expire in one hour diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-youtube-analytics/main.py b/airbyte-integrations/connectors/source-youtube-analytics/main.py index f2542cccc965d..959f6b2714542 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/main.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/main.py @@ -4,5 +4,6 @@ from source_youtube_analytics.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py index 4a9041f1f0788..cd6af9e03d9bc 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/source_youtube_analytics/source.py @@ -12,6 +12,7 @@ import pendulum import requests + from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream diff --git a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py index dcac2eb7e98ae..9744592d3ba5e 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/unit_tests/test_source.py @@ -6,9 +6,10 @@ import os from unittest.mock import MagicMock -from airbyte_cdk.sources.streams.http.auth.core import NoAuth from source_youtube_analytics.source import SourceYoutubeAnalytics +from airbyte_cdk.sources.streams.http.auth.core import NoAuth + def test_check_connection(requests_mock): access_token = "token" diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py index 13626e17bbbcd..bbcb318d24a81 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING + if TYPE_CHECKING: from dagger import Container diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py index 43ce950d77caa..72132012aaedb 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/main.py b/airbyte-integrations/connectors/source-zendesk-chat/main.py index c2c8d74d092a6..4548150f41155 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/main.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/main.py @@ -4,5 +4,6 @@ from source_zendesk_chat.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py index 2dffe978edfb7..28e6de3dd4fca 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py @@ -8,6 +8,7 @@ import pendulum import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.types import Record diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index 9c3eb3109f52b..cce977551ac77 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py index 284325c12e3b0..6cbdc2694790b 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -6,6 +6,7 @@ from typing import Any, List, Mapping, Optional, Union import requests + from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 2b0540f7cd8f1..bbaafa3751460 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py index c48196cfa1edd..8083a1cc5e0d3 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py @@ -12,25 +12,14 @@ def config() -> Mapping[str, Any]: return { "start_date": "2020-10-01T00:00:00Z", "subdomain": "", - "credentials": { - "credentials": "access_token", - "access_token": "__access_token__" - } + "credentials": {"credentials": "access_token", "access_token": "__access_token__"}, } @pytest.fixture def bans_stream_record() -> Mapping[str, Any]: return { - "ip_address": [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - } - ], + "ip_address": [{"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}], "visitor": [ { "type": "visitor", @@ -38,28 +27,22 @@ def bans_stream_record() -> Mapping[str, Any]: "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", } - ] + ], } @pytest.fixture def bans_stream_record_extractor_expected_output() -> List[Mapping[str, Any]]: return [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - }, + {"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}, { "type": "visitor", "id": 4444, "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", }, ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py index 60ae75a17da15..d33b2302e1616 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py @@ -8,8 +8,8 @@ def test_bans_stream_record_extractor( config, - requests_mock, - bans_stream_record, + requests_mock, + bans_stream_record, bans_stream_record_extractor_expected_output, ) -> None: test_url = f"https://{config['subdomain']}.zendesk.com/api/v2/chat/bans" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py index 9557a312b6359..75dad4a8f0d54 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -9,26 +9,24 @@ def _get_cursor(config) -> ZendeskChatIdIncrementalCursor: return ZendeskChatIdIncrementalCursor( - config = config, - cursor_field = "id", - field_name = "since_id", - parameters = {}, + config=config, + cursor_field="id", + field_name="since_id", + parameters={}, ) @pytest.mark.parametrize( "stream_state, expected_cursor_value, expected_state_value", [ - ({"id": 10}, 10, {'id': 10}), + ({"id": 10}, 10, {"id": 10}), ], - ids=[ - "SET Initial State and GET State" - ] + ids=["SET Initial State and GET State"], ) def test_id_incremental_cursor_set_initial_state_and_get_stream_state( - config, + config, stream_state, - expected_cursor_value, + expected_cursor_value, expected_state_value, ) -> None: cursor = _get_cursor(config) @@ -44,17 +42,14 @@ def test_id_incremental_cursor_set_initial_state_and_get_stream_state( ({"id": 123}, 123), ({"id": 456}, 456), ], - ids=[ - "first", - "second" - ] + ids=["first", "second"], ) def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: cursor = _get_cursor(config) cursor.observe(stream_slice={}, record=test_record) cursor.close_slice(stream_slice={}) assert cursor._cursor == expected - + @pytest.mark.parametrize( "stream_state, input_slice, expected", @@ -62,17 +57,14 @@ def test_id_incremental_cursor_close_slice(config, test_record, expected) -> Non ({}, {"id": 1}, {}), ({"id": 2}, {"id": 1}, {"since_id": 2}), ], - ids=[ - "No State", - "With State" - ] + ids=["No State", "With State"], ) def test_id_incremental_cursor_get_request_params(config, stream_state, input_slice, expected) -> None: cursor = _get_cursor(config) if stream_state: cursor.set_initial_state(stream_state) assert cursor.get_request_params(stream_slice=input_slice) == expected - + @pytest.mark.parametrize( "stream_state, record, expected", @@ -85,7 +77,7 @@ def test_id_incremental_cursor_get_request_params(config, stream_state, input_sl "No State", "With State > Record value", "With State < Record value", - ] + ], ) def test_id_incremental_cursor_should_be_synced(config, stream_state, record, expected) -> None: cursor = _get_cursor(config) @@ -107,7 +99,7 @@ def test_id_incremental_cursor_should_be_synced(config, stream_state, record, ex "First < Second - should not be synced", "Has First but no Second - should be synced", "Has no First and has no Second - should not be synced", - ] + ], ) def test_id_incremental_cursor_is_greater_than_or_equal(config, first_record, second_record, expected) -> None: cursor = _get_cursor(config) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py index ddf84932adc0d..48f06329adbd8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py @@ -10,19 +10,16 @@ def _get_paginator(config, id_field) -> ZendeskChatIdOffsetIncrementPaginationStrategy: return ZendeskChatIdOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - id_field = id_field, - parameters = {}, + config=config, + page_size=1, + id_field=id_field, + parameters={}, ) @pytest.mark.parametrize( "id_field, last_records, expected", - [ - ("id", [{"id": 1}], 2), - ("id", [], None) - ], + [("id", [{"id": 1}], 2), ("id", [], None)], ) def test_id_offset_increment_pagination_next_page_token(requests_mock, config, id_field, last_records, expected) -> None: paginator = _get_paginator(config, id_field) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py index 61d793cbc9f96..10838351b5c5d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py @@ -10,18 +10,18 @@ def _get_paginator(config, time_field_name) -> ZendeskChatTimeOffsetIncrementPaginationStrategy: return ZendeskChatTimeOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - time_field_name = time_field_name, - parameters = {}, + config=config, + page_size=1, + time_field_name=time_field_name, + parameters={}, ) @pytest.mark.parametrize( "time_field_name, response, last_records, expected", [ - ("end_time", {"chats":[{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), - ("end_time", {"chats":[], "end_time": 3}, [], None), + ("end_time", {"chats": [{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), + ("end_time", {"chats": [], "end_time": 3}, [], None), ], ) def test_time_offset_increment_pagination_next_page_token(requests_mock, config, time_field_name, response, last_records, expected) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py index a98cc8283e930..2528f7a5db7e1 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -4,23 +4,24 @@ import pytest -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from source_zendesk_chat.components.timestamp_based_cursor import ZendeskChatTimestampCursor +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType + def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampCursor: cursor = ZendeskChatTimestampCursor( - start_datetime = "2020-10-01T00:00:00Z", - cursor_field = cursor_field, - datetime_format = "%s", - config = config, - parameters = {}, - use_microseconds = f"{{{ {use_microseconds} }}}", + start_datetime="2020-10-01T00:00:00Z", + cursor_field=cursor_field, + datetime_format="%s", + config=config, + parameters={}, + use_microseconds=f"{{{ {use_microseconds} }}}", ) # patching missing parts cursor.start_time_option = RequestOption( - field_name = cursor_field, - inject_into = RequestOptionType.request_parameter, + field_name=cursor_field, + inject_into=RequestOptionType.request_parameter, parameters={}, ) return cursor @@ -29,25 +30,25 @@ def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampC @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), + (True, {"start_time": 1}, {"start_time": 1000000}), ], ) def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) test_result = cursor.add_microseconds({}, input_slice) assert test_result == expected - + @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), - (False, {"start_time": 1}, {'start_time': 1}), + (True, {"start_time": 1}, {"start_time": 1000000}), + (False, {"start_time": 1}, {"start_time": 1}), ], ids=[ "WITH `use_microseconds`", "WITHOUT `use_microseconds`", - ] + ], ) def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py index 9e6409236281f..a612c74fc689e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zendesk-support/main.py b/airbyte-integrations/connectors/source-zendesk-support/main.py index 88eed5ec56afa..4577f470a2dc1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/main.py +++ b/airbyte-integrations/connectors/source-zendesk-support/main.py @@ -4,5 +4,6 @@ from source_zendesk_support.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py index 1f631c62a1303..4d205ed5e83b0 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType @@ -33,7 +34,9 @@ def get_request_params( start_time = stream_slice.get(self._partition_field_start.eval(self.config)) options[self.start_time_option.field_name.eval(config=self.config)] = [start_time] # type: ignore # field_name is always casted to an interpolated string if self.end_time_option and self.end_time_option.inject_into == option_type: - options[self.end_time_option.field_name.eval(config=self.config)].append(stream_slice.get(self._partition_field_end.eval(self.config))) # type: ignore # field_name is always casted to an interpolated string + options[self.end_time_option.field_name.eval(config=self.config)].append( + stream_slice.get(self._partition_field_end.eval(self.config)) + ) # type: ignore # field_name is always casted to an interpolated string return options diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py index 0df09fb2f3756..d040b0f779752 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py @@ -7,10 +7,11 @@ from datetime import datetime from typing import List +from orjson import orjson + from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch, logger from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteMessageSerializer, AirbyteTraceMessage, TraceType, Type -from orjson import orjson from source_zendesk_support import SourceZendeskSupport diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py index 15d04a0d21dc8..f7d4746facf6d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py @@ -8,6 +8,7 @@ from typing import Any, List, Mapping, Optional, Tuple import pendulum + from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.source import TState @@ -29,6 +30,7 @@ UserSettingsStream, ) + logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 911c090f55456..8cc0026bd99f7 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -13,6 +13,7 @@ import pendulum import pytz import requests + from airbyte_cdk import BackoffStrategy from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration @@ -24,6 +25,7 @@ from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException + DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY: str = "_last_end_time" END_OF_STREAM_KEY: str = "end_of_stream" @@ -518,7 +520,6 @@ def migrate(self, stream_state: Optional[Mapping[str, Any]]) -> Mapping[str, Any class TicketMetrics(SourceZendeskSupportStream): - name = "ticket_metrics" cursor_field = "_ab_updated_at" should_checkpoint = False @@ -583,7 +584,6 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, class StatelessTicketMetrics(FullRefreshZendeskSupportStream): - response_list_name: str = "ticket_metrics" cursor_field: str = "updated_at" should_checkpoint = False @@ -631,7 +631,6 @@ def _get_updated_state(self, current_stream_state: Mapping[str, Any], latest_rec class StatefulTicketMetrics(HttpSubStream, IncrementalZendeskSupportStream): - response_list_name: str = "ticket_metric" _state_cursor_field: str = "_ab_updated_at" _legacy_cursor_field: str = "generated_timestamp" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index c3d9c1c98188f..27703dc2ddbb3 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -2,4 +2,5 @@ import os + os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py index 5f02c2b74fbbf..a2f658a70079b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py @@ -1,9 +1,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import pendulum +from pendulum.datetime import DateTime + from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath -from pendulum.datetime import DateTime from .utils import datetime_to_string from .zs_requests import ( @@ -81,30 +82,28 @@ def given_post_comments( ) return post_comments_record_builder + def given_tickets(http_mocker: HttpMocker, start_date: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: """ Tickets requests setup """ - tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field( - FieldPath("generated_timestamp"), start_date.int_timestamp - ) + tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field(FieldPath("generated_timestamp"), start_date.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder -def given_tickets_with_state(http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: + +def given_tickets_with_state( + http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator +) -> TicketsRecordBuilder: """ Tickets requests setup """ tickets_record_builder = TicketsRecordBuilder.tickets_record().with_cursor(cursor_value.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder @@ -114,24 +113,20 @@ def given_groups_with_later_records( http_mocker: HttpMocker, updated_at_value: DateTime, later_record_time_delta: pendulum.duration, - api_token_authenticator: ApiTokenAuthenticator + api_token_authenticator: ApiTokenAuthenticator, ) -> GroupsRecordBuilder: """ Creates two group records one with a specific cursor value and one that has a later cursor value based on the provided timedelta. This is intended to create multiple records with different times which can be used to test functionality like semi-incremental record filtering """ - groups_record_builder = GroupsRecordBuilder.groups_record().with_field( - FieldPath("updated_at"), datetime_to_string(updated_at_value) - ) + groups_record_builder = GroupsRecordBuilder.groups_record().with_field(FieldPath("updated_at"), datetime_to_string(updated_at_value)) later_groups_record_builder = GroupsRecordBuilder.groups_record().with_field( FieldPath("updated_at"), datetime_to_string(updated_at_value + later_record_time_delta) ) http_mocker.get( - GroupsRequestBuilder.groups_endpoint(api_token_authenticator) - .with_page_size(100) - .build(), + GroupsRequestBuilder.groups_endpoint(api_token_authenticator).with_page_size(100).build(), GroupsResponseBuilder.groups_response().with_record(groups_record_builder).with_record(later_groups_record_builder).build(), ) return groups_record_builder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py index 032b9ed1d3535..e672521e1501a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py @@ -4,6 +4,7 @@ from unittest import TestCase import pendulum + from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.state_builder import StateBuilder @@ -13,6 +14,7 @@ from .utils import datetime_to_string, read_stream, string_to_datetime from .zs_requests.request_authenticators import ApiTokenAuthenticator + _NOW = datetime.now(timezone.utc) @@ -62,9 +64,7 @@ def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_reco api_token_authenticator, ) - state_value = { - "updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50)) - } + state_value = {"updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50))} state = StateBuilder().with_stream_state("groups", state_value).build() diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py index ffe141cdfcb4f..1755a17c7f6e5 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostCommentVotesResponseBuilder from .zs_responses.records import PostCommentVotesRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -96,7 +97,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -126,7 +133,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py index a9d2d825c5bb0..a2d140bcf2c34 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostsCommentsResponseBuilder from .zs_responses.records import PostsCommentsRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -84,7 +85,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +116,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py index a2617b5a07df1..8ca10b18980dd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py @@ -6,9 +6,9 @@ import freezegun import pendulum -from airbyte_cdk.models import AirbyteStateBlob + +from airbyte_cdk.models import AirbyteStateBlob, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath from airbyte_cdk.test.state_builder import StateBuilder @@ -21,6 +21,7 @@ from .zs_responses import ErrorResponseBuilder, PostsVotesResponseBuilder from .zs_responses.records import PostsVotesRecordBuilder + _NOW = datetime.now(timezone.utc) @@ -84,7 +85,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +116,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py index 699d09742f5df..c5e2dda402cdf 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py @@ -4,6 +4,7 @@ import freezegun import pendulum + from airbyte_cdk.models.airbyte_protocol import AirbyteStateBlob, SyncMode from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import FieldPath @@ -17,20 +18,21 @@ from .zs_responses import TicketMetricsResponseBuilder from .zs_responses.records import TicketMetricsRecordBuilder + _NOW = pendulum.now(tz="UTC") _TWO_YEARS_AGO_DATETIME = _NOW.subtract(years=2) + @freezegun.freeze_time(_NOW.isoformat()) class TestTicketMetricsIncremental(TestCase): - @property def _config(self): return ( ConfigBuilder() - .with_basic_auth_credentials("user@example.com", "password") - .with_subdomain("d3v-airbyte") - .with_start_date(_TWO_YEARS_AGO_DATETIME) - .build() + .with_basic_auth_credentials("user@example.com", "password") + .with_subdomain("d3v-airbyte") + .with_start_date(_TWO_YEARS_AGO_DATETIME) + .build() ) def _get_authenticator(self, config): @@ -46,7 +48,7 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec http_mocker.get( TicketMetricsRequestBuilder.stateless_ticket_metrics_endpoint(api_token_authenticator).with_page_size(100).build(), - TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build() + TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) @@ -55,7 +57,6 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec assert output.most_recent_state.stream_descriptor.name == "ticket_metrics" assert output.most_recent_state.stream_state.__dict__ == {"_ab_updated_at": pendulum.parse(record_updated_at).int_timestamp} - @HttpMocker() def test_given_state_and_successful_sync_when_read_then_return_record(self, http_mocker): api_token_authenticator = self._get_authenticator(self._config) @@ -63,16 +64,20 @@ def test_given_state_and_successful_sync_when_read_then_return_record(self, http state_cursor_value = pendulum.now(tz="UTC").subtract(days=2).int_timestamp state = StateBuilder().with_stream_state("ticket_metrics", state={"_ab_updated_at": state_cursor_value}).build() record_cursor_value = pendulum.now(tz="UTC").subtract(days=1) - tickets_records_builder = given_tickets_with_state(http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value,api_token_authenticator) + tickets_records_builder = given_tickets_with_state( + http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value, api_token_authenticator + ) ticket = tickets_records_builder.build() - ticket_metrics_first_record_builder = TicketMetricsRecordBuilder.stateful_ticket_metrics_record().with_field( - FieldPath("ticket_id"), ticket["id"] - ).with_cursor(ticket["generated_timestamp"]) + ticket_metrics_first_record_builder = ( + TicketMetricsRecordBuilder.stateful_ticket_metrics_record() + .with_field(FieldPath("ticket_id"), ticket["id"]) + .with_cursor(ticket["generated_timestamp"]) + ) http_mocker.get( TicketMetricsRequestBuilder.stateful_ticket_metrics_endpoint(api_token_authenticator, ticket["id"]).build(), - TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build() + TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py index d3185b94d95e1..aa19dc6ef9cae 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py @@ -4,13 +4,13 @@ from typing import Any, Dict, List, Optional import pendulum -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage +from pendulum.datetime import DateTime +from source_zendesk_support import SourceZendeskSupport + +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, SyncMode from airbyte_cdk.models import Level as LogLevel -from airbyte_cdk.models import SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from pendulum.datetime import DateTime -from source_zendesk_support import SourceZendeskSupport def read_stream( diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py index a185dd225a2f1..8d72e988f203e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class ZendeskSupportBaseRequestBuilder(ZendeskSuppportRequestBuilder): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py index 3d224cd794564..94629bb155266 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py @@ -10,7 +10,9 @@ class PostCommentVotesRequestBuilder(ZendeskSupportBaseRequestBuilder): @classmethod - def post_comment_votes_endpoint(cls, authenticator: Authenticator, post_id: int, post_comment_id: int) -> "PostCommentVotesRequestBuilder": + def post_comment_votes_endpoint( + cls, authenticator: Authenticator, post_id: int, post_comment_id: int + ) -> "PostCommentVotesRequestBuilder": return cls("d3v-airbyte", f"community/posts/{post_id}/comments/{post_comment_id}/votes").with_authenticator(authenticator) def __init__(self, subdomain: str, resource: str) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py index 08d634cea66c4..adef1134aadb9 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py @@ -10,6 +10,7 @@ from source_zendesk_support.source import SourceZendeskSupport from source_zendesk_support.streams import Users + _ANY_ATTEMPT_COUNT = 10 diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py index 6398c165cc460..c0b61a2d29f4d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_components.py @@ -4,13 +4,14 @@ import pytest import requests -from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from source_zendesk_support.components import ( ZendeskSupportAttributeDefinitionsExtractor, ZendeskSupportAuditLogsIncrementalSync, ZendeskSupportExtractorEvents, ) +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType + @pytest.mark.parametrize( "stream_state, stream_slice, next_page_token, expected_params", diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index ae922ca7ffc81..7c84300b4d879 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -16,8 +16,6 @@ import pytest import pytz import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction from source_zendesk_support.source import BasicApiTokenAuthenticator, SourceZendeskSupport from source_zendesk_support.streams import ( DATETIME_FORMAT, @@ -65,6 +63,10 @@ from test_data.data import TICKET_EVENTS_STREAM_RESPONSE from utils import read_full_refresh +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http.error_handlers import ResponseAction + + TICKET_SUBSTREAMS = [StatefulTicketMetrics] # prepared config @@ -143,7 +145,9 @@ def test_convert_config2stream_args(config): @freezegun.freeze_time("2022-01-01") def test_default_start_date(): - result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args(TEST_CONFIG_WITHOUT_START_DATE) + result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args( + TEST_CONFIG_WITHOUT_START_DATE + ) assert result["start_date"] == "2020-01-01T00:00:00Z" @@ -1047,14 +1051,14 @@ def test_read_non_json_error(requests_mock, caplog): read_full_refresh(stream) assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") -class TestTicketMetrics: +class TestTicketMetrics: @pytest.mark.parametrize( - "state, expected_implemented_stream", - [ - ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - ({}, StatelessTicketMetrics), - ] + "state, expected_implemented_stream", + [ + ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + ({}, StatelessTicketMetrics), + ], ) def test_get_implemented_stream(self, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1062,12 +1066,12 @@ def test_get_implemented_stream(self, state, expected_implemented_stream): assert isinstance(implemented_stream, expected_implemented_stream) @pytest.mark.parametrize( - "sync_mode, state, expected_implemented_stream", - [ - (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - (SyncMode.full_refresh, {}, StatelessTicketMetrics), - (SyncMode.incremental, {}, StatelessTicketMetrics), - ] + "sync_mode, state, expected_implemented_stream", + [ + (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + (SyncMode.full_refresh, {}, StatelessTicketMetrics), + (SyncMode.incremental, {}, StatelessTicketMetrics), + ], ) def test_stream_slices(self, sync_mode, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1081,7 +1085,12 @@ class TestStatefulTicketMetrics: [ ( {}, - {"tickets": [{"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}]}, + { + "tickets": [ + {"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + ] + }, [ {"ticket_id": "13", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"ticket_id": "80", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, @@ -1108,8 +1117,7 @@ def test_stream_slices(self, requests_mock, stream_state, response, expected_sli def test_read_with_error(self, requests_mock): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) requests_mock.get( - f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", - json={"error": "RecordNotFound", "description": "Not found"} + f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", json={"error": "RecordNotFound", "description": "Not found"} ) records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"ticket_id": "13"})) @@ -1119,12 +1127,12 @@ def test_read_with_error(self, requests_mock): @pytest.mark.parametrize( "status_code, response_action", ( - (200, ResponseAction.SUCCESS), - (404, ResponseAction.IGNORE), - (403, ResponseAction.IGNORE), - (500, ResponseAction.RETRY), - (429, ResponseAction.RATE_LIMITED), - ) + (200, ResponseAction.SUCCESS), + (404, ResponseAction.IGNORE), + (403, ResponseAction.IGNORE), + (500, ResponseAction.RETRY), + (429, ResponseAction.RATE_LIMITED), + ), ) def test_should_retry(self, status_code: int, response_action: bool): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) @@ -1135,37 +1143,62 @@ def test_should_retry(self, status_code: int, response_action: bool): @pytest.mark.parametrize( "current_stream_state, record_cursor_value, expected", [ - ({ "_ab_updated_at": 1727334000}, 1727420400, { "_ab_updated_at": 1727420400}), - ({ "_ab_updated_at": 1727334000}, 1700000000, { "_ab_updated_at": 1727334000}), - ] + ({"_ab_updated_at": 1727334000}, 1727420400, {"_ab_updated_at": 1727420400}), + ({"_ab_updated_at": 1727334000}, 1700000000, {"_ab_updated_at": 1727334000}), + ], ) def test_get_updated_state(self, current_stream_state, record_cursor_value, expected): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) - latest_record = { "id": 1, "_ab_updated_at": record_cursor_value} + latest_record = {"id": 1, "_ab_updated_at": record_cursor_value} output_state = stream._get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) assert output_state == expected class TestStatelessTicketMetrics: @pytest.mark.parametrize( - "start_date, response, expected", - [ - ( - "2023-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp}, - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + "start_date, response, expected", + [ + ( + "2023-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ), - ( - "2024-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + }, + [ + { + "id": 1, + "ticket_id": 999, + "updated_at": "2023-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp, + }, + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + }, + ], + ), + ( + "2024-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ) - ] + }, + [ + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } + ], + ), + ], ) def test_parse_response(self, requests_mock, start_date, response, expected): stream_args = copy.deepcopy(STREAM_ARGS) @@ -1176,25 +1209,24 @@ def test_parse_response(self, requests_mock, start_date, response, expected): output = list(stream.parse_response(test_response, {})) assert expected == output - @pytest.mark.parametrize( - "has_more, expected", - [ - (True, {"page[after]": "nextpagecursor"}), - (False, None) - ] - ) + @pytest.mark.parametrize("has_more, expected", [(True, {"page[after]": "nextpagecursor"}), (False, None)]) def test_next_page_token(self, mocker, has_more, expected): stream = StatelessTicketMetrics(**STREAM_ARGS) ticket_metrics_response = mocker.Mock() - ticket_metrics_response.json.return_value = {"meta": { "after_cursor": "nextpagecursor", "has_more": has_more}} + ticket_metrics_response.json.return_value = {"meta": {"after_cursor": "nextpagecursor", "has_more": has_more}} result = stream.next_page_token(response=ticket_metrics_response) assert expected == result def test_get_updated_state(self): stream = StatelessTicketMetrics(**STREAM_ARGS) - stream._most_recently_updated_record = {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + stream._most_recently_updated_record = { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } output_state = stream._get_updated_state(current_stream_state={}, latest_record={}) - expected_state = { "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + expected_state = {"_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} assert output_state == expected_state @@ -1236,13 +1268,7 @@ def test_validate_response_ticket_audits_handle_empty_response(audits_response, assert stream._validate_response(response_mock, {}) == expected -@pytest.mark.parametrize( - "initial_state_cursor_field", - [ - "generated_timestamp", - "_ab_updated_at" - ] -) +@pytest.mark.parametrize("initial_state_cursor_field", ["generated_timestamp", "_ab_updated_at"]) def test_ticket_metrics_state_migrataion(initial_state_cursor_field): state_migrator = TicketMetricsStateMigration() initial_state = {initial_state_cursor_field: 1672531200} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/components.py b/airbyte-integrations/connectors/source-zendesk-talk/components.py index d8030841afe21..f9f3b12732a0e 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/components.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/components.py @@ -4,6 +4,7 @@ from typing import Any, List, Mapping import requests + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zenloop/main.py b/airbyte-integrations/connectors/source-zenloop/main.py index dd3a6687740ec..502dfc3af95f2 100644 --- a/airbyte-integrations/connectors/source-zenloop/main.py +++ b/airbyte-integrations/connectors/source-zenloop/main.py @@ -4,5 +4,6 @@ from source_zenloop.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py index 2f85e4b1b8e11..a2bec37817984 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py @@ -12,7 +12,6 @@ @dataclass class ZenloopPartitionRouter(SubstreamPartitionRouter): - config: Config def stream_slices(self) -> Iterable[StreamSlice]: diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py index 15a603417b4bf..70f571dc05438 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/source.py @@ -4,6 +4,7 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into source connector. diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py index 00f466edd7e31..53ece70f20ff5 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/streams.py @@ -9,11 +9,11 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpStream class ZenloopStream(HttpStream, ABC): - url_base = "https://api.zenloop.com/v1/" extra_params = None has_date_param = False @@ -58,7 +58,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class ChildStreamMixin: - parent_stream_class: Optional[ZenloopStream] = None def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: diff --git a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py index 472f2799e63d4..76d7794113f89 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py +++ b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/test_stream_factory.py @@ -11,6 +11,7 @@ import requests from source_zoho_crm.streams import IncrementalZohoCrmStream, ZohoStreamFactory + HERE = Path(__file__).parent diff --git a/airbyte-integrations/connectors/source-zoho-crm/main.py b/airbyte-integrations/connectors/source-zoho-crm/main.py index 2cf82bb23bf9a..d04f943d86bf2 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/main.py +++ b/airbyte-integrations/connectors/source-zoho-crm/main.py @@ -4,5 +4,6 @@ from source_zoho_crm.run import run + if __name__ == "__main__": run() diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index 15425f380be42..b9ebc28e9f6a2 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -5,6 +5,7 @@ from setuptools import find_packages, setup + MAIN_REQUIREMENTS = [ "airbyte-cdk", ] diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py index edb92e2c8e5d8..abaf9d55c7616 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/api.py @@ -11,6 +11,7 @@ from .auth import ZohoOauth2Authenticator + logger = logging.getLogger(__name__) diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py index f6cde3b112108..6f47fddbee115 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/auth.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Mapping, Tuple import requests + from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py index d915dc05f2b9f..05ca883b8f7ef 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/run.py @@ -5,9 +5,10 @@ import sys -from airbyte_cdk.entrypoint import launch from source_zoho_crm import SourceZohoCrm +from airbyte_cdk.entrypoint import launch + def run(): source = SourceZohoCrm() diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py index d6ead32026f83..f5f4ee55e4707 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/source.py @@ -10,6 +10,7 @@ from .api import ZohoAPI from .streams import ZohoStreamFactory + if TYPE_CHECKING: # This is a workaround to avoid circular import in the future. # TYPE_CHECKING is False at runtime, but True when system performs type checking diff --git a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py index ad6b268683961..53df5ab1e4283 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py +++ b/airbyte-integrations/connectors/source-zoho-crm/source_zoho_crm/streams.py @@ -11,12 +11,14 @@ from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional import requests + from airbyte_cdk.sources.streams.http import HttpStream from .api import ZohoAPI from .exceptions import IncompleteMetaDataException, UnknownDataTypeException from .types import FieldMeta, ModuleMeta, ZohoPickListItem + # 204 and 304 status codes are valid successful responses, # but `.json()` will fail because the response body is empty EMPTY_BODY_STATUSES = (HTTPStatus.NO_CONTENT, HTTPStatus.NOT_MODIFIED) diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py index 31ad48a1a41cd..ca796e341b231 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py @@ -6,6 +6,7 @@ import pytest + TestCase = namedtuple( "TestCase", ("json_type", "data_type", "length", "decimal_place", "api_name", "pick_list_values", "autonumber", "expected_values") ) diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py index 0fea743d100d2..a32f0adf21405 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_auth.py @@ -6,6 +6,7 @@ from source_zoho_crm.auth import ZohoOauth2Authenticator + authenticator = ZohoOauth2Authenticator("http://dummy.url/oauth/v2/token", "client_id", "client_secret", "refresh_token") diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py index 3774064c60a28..e761736c840b5 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_incremental_streams.py @@ -5,9 +5,10 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import SyncMode from source_zoho_crm.streams import IncrementalZohoCrmStream as BaseIncrementalZohoCrmStream +from airbyte_cdk.models import SyncMode + @pytest.fixture def stream_factory(mocker): diff --git a/airbyte-integrations/connectors/source-zoom/components.py b/airbyte-integrations/connectors/source-zoom/components.py index 00214c737833e..ef268f7745503 100644 --- a/airbyte-integrations/connectors/source-zoom/components.py +++ b/airbyte-integrations/connectors/source-zoom/components.py @@ -9,10 +9,12 @@ from typing import Any, Mapping, Optional, Union import requests +from requests import HTTPError + from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.types import Config -from requests import HTTPError + # https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response # The Bearer token generated by server-to-server token will expire in one hour diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py index 82823254d2666..fbed37992cb4f 100644 --- a/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/acceptance.py @@ -5,6 +5,7 @@ import pytest + pytest_plugins = ("connector_acceptance_test.plugin",) diff --git a/tools/bin/cleanup-workflow-runs.py b/tools/bin/cleanup-workflow-runs.py index d2ef9a38cbbc4..f022934571d11 100644 --- a/tools/bin/cleanup-workflow-runs.py +++ b/tools/bin/cleanup-workflow-runs.py @@ -10,6 +10,7 @@ from github import Github + DAYS_TO_KEEP_ORPHANED_JOBS = 90 diff --git a/tools/bin/identify-dormant-workflows.py b/tools/bin/identify-dormant-workflows.py index 5dfa954c60564..40a62d426af3d 100644 --- a/tools/bin/identify-dormant-workflows.py +++ b/tools/bin/identify-dormant-workflows.py @@ -13,6 +13,7 @@ from slack_sdk import WebClient from slack_sdk.errors import SlackApiError + DAYS_TO_KEEP_ORPHANED_JOBS = 90 SLACK_CHANNEL_FOR_NOTIFICATIONS = "infra-alerts" @@ -97,7 +98,6 @@ def main(): print(message) if slack_token: - print("Sending Slack notification...") client = WebClient(slack_token) diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index d044a45bebce7..93529595f7066 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -6,6 +6,7 @@ import json import os + """ This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed diff --git a/tools/bin/record_obfuscator.py b/tools/bin/record_obfuscator.py index 4dff42a71a058..031010549fb1e 100755 --- a/tools/bin/record_obfuscator.py +++ b/tools/bin/record_obfuscator.py @@ -7,6 +7,7 @@ import sys from typing import Any + # # record_obfuscator is a tiny script that: # 1. reads JSON lines from stdin diff --git a/tools/bin/update_intellij_venv.py b/tools/bin/update_intellij_venv.py index dfb259ee92551..d2474db6ac4ba 100644 --- a/tools/bin/update_intellij_venv.py +++ b/tools/bin/update_intellij_venv.py @@ -8,6 +8,7 @@ import sys import xml.etree.ElementTree as ET + INTELLIJ_VERSION_FLAG = "-intellij-version" diff --git a/tools/schema_generator/schema_generator/infer_schemas.py b/tools/schema_generator/schema_generator/infer_schemas.py index 6e01353ce59c1..bf58934480fb7 100644 --- a/tools/schema_generator/schema_generator/infer_schemas.py +++ b/tools/schema_generator/schema_generator/infer_schemas.py @@ -27,10 +27,11 @@ import sys import genson.schema.strategies as strategies -from airbyte_cdk.models import AirbyteMessage, Type from genson import SchemaBuilder from genson.schema.strategies.object import Object +from airbyte_cdk.models import AirbyteMessage, Type + class NoRequiredObj(Object): """ diff --git a/tools/schema_generator/schema_generator/main.py b/tools/schema_generator/schema_generator/main.py index bd1cb14ae8142..dd7f62bc75d08 100644 --- a/tools/schema_generator/schema_generator/main.py +++ b/tools/schema_generator/schema_generator/main.py @@ -10,7 +10,6 @@ def main(): - parser = argparse.ArgumentParser(description="Airbyte Schema Generator") if len(sys.argv) == 1: