From 5ae7cfe0e5787a619ed1ee30d050c6554d4f86b5 Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Sun, 15 Dec 2024 22:08:13 -0800 Subject: [PATCH] formatting --- .../integration_tests/test_documentation.py | 8 +- .../unit_tests/test_checks/test_assets.py | 1 - .../test_checks/test_documentation.py | 96 +-- .../unit_tests/test_checks/test_packaging.py | 1 + .../unit_tests/test_checks/test_testing.py | 36 +- .../commons/backends/base_backend.py | 3 +- .../commons/backends/file_backend.py | 6 +- .../src/live_tests/commons/proxy.py | 22 +- .../live-tests/src/live_tests/conftest.py | 10 +- .../live-tests/src/live_tests/report.py | 6 +- .../metadata_invalid_support_refreshes.yaml | 2 +- ...reaking_changes_empty_impacted_stream.yaml | 4 +- .../lib/tests/test_commands.py | 1 + .../lib/tests/test_spec_cache.py | 7 +- .../connectors/generate_erd/pipeline.py | 2 +- .../airbyte_ci/connectors/reports.py | 6 +- .../connectors/test/steps/common.py | 8 +- .../pipelines/airbyte_ci/metadata/pipeline.py | 8 +- .../utils/client_container_runner.py | 2 +- .../unit_tests/test_connector_attributes.py | 77 +- .../unit_tests/test_core.py | 112 ++- .../unit_tests/test_documentation.py | 124 ++-- .../unit_tests/test_incremental.py | 201 +++--- .../unit_tests/test_spec.py | 549 +++++++-------- .../integration_tests/integration_test.py | 12 +- .../unit_tests/indexer_test.py | 12 +- .../destination_aws_datalake/destination.py | 1 - .../integration_tests/integration_test.py | 14 +- .../destination_chroma/destination.py | 1 - .../destination_databend/destination.py | 1 - .../unit_tests/test_databend_destination.py | 32 +- .../integration_tests/integration_test.py | 86 +-- .../unit_tests/destination_unit_tests.py | 3 + .../destination_firebolt/destination.py | 1 - .../unit_tests/test_firebolt_destination.py | 2 + .../destination_google_sheets/destination.py | 1 - .../destination_google_sheets/helpers.py | 1 - .../integration_tests/integration_test.py | 1 + .../unit_tests/unit_test.py | 1 + .../destination-iceberg/finalize_build.py | 1 + .../destination_kvdb/destination.py | 1 - .../integration_tests/integration_test.py | 61 +- .../common/catalog/catalog_providers.py | 3 +- .../common/destinations/record_processor.py | 8 +- .../common/sql/sql_processor.py | 20 +- .../destination_pgvector/destination.py | 5 +- .../pgvector_processor.py | 5 +- .../integration_tests/integration_test.py | 3 +- .../unit_tests/destination_test.py | 3 +- .../pinecone_integration_test.py | 60 +- .../unit_tests/pinecone_indexer_test.py | 29 +- .../destination_sftp_json/destination.py | 1 - .../common/catalog/catalog_providers.py | 8 +- .../common/destinations/record_processor.py | 4 +- .../common/sql/sql_processor.py | 62 +- .../cortex_processor.py | 15 +- .../destination.py | 8 +- .../integration_tests/integration_test.py | 16 +- .../destination_sqlite/destination.py | 1 - .../destination_typesense/writer.py | 2 +- .../destination_vectara/destination.py | 1 - .../integration_tests/integration_test.py | 1 + .../source-adjust/unit_tests/conftest.py | 17 +- .../source-adjust/unit_tests/test_streams.py | 8 +- .../test_airtable_backoff_strategy.py | 8 +- .../unit_tests/test_airtable_error_handler.py | 17 +- .../unit_tests/conftest.py | 5 +- .../integration/test_report_based_streams.py | 10 +- .../unit_tests/integration/utils.py | 6 +- .../unit_tests/test_finance_streams.py | 10 +- .../unit_tests/test_source.py | 20 +- .../unit_tests/test_streams.py | 10 +- .../unit_tests/test_custom_extractors.py | 4 +- .../unit_tests/test_config_migrations.py | 10 +- .../source-avni/unit_tests/test_components.py | 25 +- .../config_migrations.py | 6 +- .../unit_tests/test_authenticator.py | 12 +- .../unit_tests/test_stream_reader.py | 29 +- .../unit_tests/test_azure_table.py | 22 +- .../test_app_install_ad_labels_stream.py | 18 +- .../test_app_install_ads_stream.py | 16 +- .../unit_tests/test_bulk_streams.py | 54 +- .../unit_tests/test_reports.py | 54 +- .../unit_tests/integration/config.py | 4 +- .../unit_tests/integration/pagination.py | 2 +- .../unit_tests/integration/request_builder.py | 15 +- .../integration/response_builder.py | 3 +- .../unit_tests/integration/test_addon.py | 51 +- .../unit_tests/integration/test_coupon.py | 35 +- .../unit_tests/integration/test_customer.py | 50 +- .../unit_tests/integration/test_event.py | 29 +- .../integration/test_hosted_page.py | 43 +- .../unit_tests/integration/test_plan.py | 47 +- .../integration/test_site_migration_detail.py | 54 +- .../integration/test_subscription.py | 49 +- ...est_subscription_with_scheduled_changes.py | 75 +- .../integration/test_virtual_bank_account.py | 49 +- .../unit_tests/test_component.py | 29 +- .../source-commcare/source_commcare/source.py | 1 - .../test_source_declarative_local_manifest.py | 12 +- .../streams/base_insight_streams.py | 8 +- .../streams/common.py | 3 +- .../test_ads_insights_action_product_id.py | 232 +++--- .../unit_tests/integration/test_videos.py | 14 +- .../unit_tests/integration/utils.py | 2 +- .../unit_tests/test_base_insight_streams.py | 241 ++++--- .../unit_tests/test_client.py | 4 +- .../unit_tests/test_config_migrations.py | 7 +- .../unit_tests/test_errors.py | 102 +-- .../test_custom_field_transformation.py | 7 +- .../source-file/integration_tests/conftest.py | 3 +- .../source-file/unit_tests/test_client.py | 9 +- .../unit_tests/test_firebolt_source.py | 4 + .../source-gcs/integration_tests/conftest.py | 1 - .../source-gcs/unit_tests/conftest.py | 3 +- .../source-gcs/unit_tests/test_config.py | 1 - .../unit_tests/test_config_migrations.py | 15 +- .../source-gcs/unit_tests/test_cursor.py | 5 +- .../source-gcs/unit_tests/test_stream.py | 17 +- .../unit_tests/test_stream_reader.py | 5 +- .../source-gcs/unit_tests/test_zip_helper.py | 3 +- .../source_github/config_migrations.py | 6 +- .../unit_tests/integration/test_assignees.py | 19 +- .../source-github/unit_tests/test_stream.py | 48 +- .../source_gitlab/config_migrations.py | 6 +- .../unit_tests/test_partition_routers.py | 4 +- .../source-gitlab/unit_tests/test_source.py | 8 +- .../source-gitlab/unit_tests/test_streams.py | 5 +- .../source-gitlab/unit_tests/test_utils.py | 2 +- .../unit_tests/test_request_with_filter.py | 28 +- .../source-google-ads/unit_tests/conftest.py | 1 + .../unit_tests/test_errors.py | 5 +- .../unit_tests/test_google_ads.py | 26 +- .../test_incremental_events_streams.py | 36 +- .../unit_tests/test_streams.py | 6 +- .../unit_tests/test_api_quota.py | 2 +- .../unit_tests/test_source.py | 58 +- .../unit_tests/test_streams.py | 68 +- .../unit_tests/test_utils.py | 4 +- .../unit_tests/unit_test.py | 4 +- .../unit_tests/conftest.py | 5 +- .../integration/custom_http_mocker.py | 23 +- .../unit_tests/integration/request_builder.py | 4 +- .../integration/test_credentials.py | 4 +- .../unit_tests/integration/test_source.py | 208 ++++-- .../unit_tests/test_stream.py | 45 +- .../source-greenhouse/unit_tests/conftest.py | 9 +- .../unit_tests/test_components.py | 69 +- .../source-hubspot/unit_tests/conftest.py | 9 +- .../unit_tests/integrations/config_builder.py | 4 +- .../integrations/request_builders/api.py | 3 +- .../integrations/request_builders/streams.py | 38 +- .../contact_response_builder.py | 1 - .../integrations/response_builder/helpers.py | 2 +- .../response_builder/pagination.py | 11 +- .../integrations/response_builder/streams.py | 2 +- .../test_contacts_form_submissions.py | 152 ++-- .../test_contacts_list_memberships.py | 85 ++- .../test_contacts_merged_audit.py | 129 ++-- .../integrations/test_engagements_calls.py | 39 +- .../unit_tests/integrations/test_leads.py | 38 +- .../integrations/test_owners_archived.py | 11 +- .../test_web_analytics_streams.py | 136 ++-- .../unit_tests/test_components.py | 36 +- .../source-hubspot/unit_tests/test_source.py | 16 +- .../source-hubspot/unit_tests/test_streams.py | 67 +- .../integration_tests/test_streams.py | 12 +- .../source-instagram/unit_tests/conftest.py | 9 +- .../unit_tests/integration/request_builder.py | 1 - .../integration/response_builder.py | 19 +- .../unit_tests/integration/test_api.py | 10 +- .../unit_tests/integration/test_media.py | 48 +- .../integration/test_media_insights.py | 128 ++-- .../unit_tests/integration/test_stories.py | 8 +- .../integration/test_story_insights.py | 46 +- .../test_user_lifetime_insights.py | 10 +- .../unit_tests/integration/test_users.py | 12 +- .../source-instagram/unit_tests/records.py | 659 +++++------------- .../unit_tests/test_source.py | 18 +- .../unit_tests/test_streams.py | 2 - .../source-iterable/source_iterable/source.py | 1 + .../unit_tests/test_config_migrations.py | 2 +- .../source-jira/source_jira/streams.py | 1 - .../unit_tests/integration/test_issues.py | 25 +- .../source-jira/unit_tests/test_streams.py | 118 ++-- .../unit_tests/integration/config.py | 2 +- .../unit_tests/integration/test_profiles.py | 10 +- .../unit_tests/test_included_extractor.py | 22 +- .../test_per_partition_state_migration.py | 20 +- .../source-klaviyo/unit_tests/test_source.py | 8 +- .../source-klaviyo/unit_tests/test_streams.py | 59 +- .../unit_tests/test_bank_balances_stream.py | 7 +- .../source-kyriba/unit_tests/test_source.py | 1 + .../source-kyve/source_kyve/source.py | 2 +- .../unit_tests/test_components.py | 8 +- .../unit_tests/test_source.py | 38 +- .../unit_tests/test_streams.py | 3 +- .../samples/test_data_for_tranform.py | 4 +- .../utils_tests/test_update_specific_key.py | 4 +- .../source-looker/source_looker/components.py | 1 - .../source-marketo/unit_tests/conftest.py | 4 +- .../source-marketo/unit_tests/test_source.py | 38 +- .../unit_tests/unit_tests.py | 8 +- .../unit_tests/test_stream_reader.py | 15 +- .../unit_tests/test_utils.py | 7 +- .../source-mixpanel/unit_tests/conftest.py | 2 + .../source-mixpanel/unit_tests/test_source.py | 15 +- .../unit_tests/test_streams.py | 360 +++++----- .../source-mixpanel/unit_tests/utils.py | 4 +- .../monday_requests/base_requests_builder.py | 7 +- .../error_response_builder.py | 1 - .../unit_tests/test_graphql_requester.py | 10 +- .../integration_tests/seed/hook.py | 57 +- .../unit_tests/test_components.py | 136 ++-- .../unit_tests/test_python_streams.py | 162 ++++- .../source-notion/unit_tests/test_source.py | 3 +- .../connectors/source-orb/components.py | 1 - .../source_paypal_transaction/source.py | 1 + .../unit_tests/auth_components_test.py | 47 +- .../unit_tests/conftest.py | 39 +- .../unit_tests/pagination_cursor.py | 43 +- .../unit_tests/pagination_increment.py | 23 +- .../source-pinterest/unit_tests/test_auth.py | 5 +- .../unit_tests/test_incremental_streams.py | 3 +- .../unit_tests/test_reports.py | 52 +- .../unit_tests/test_source.py | 3 +- .../unit_tests/test_streams.py | 7 +- .../integration_tests/seed/hook.py | 92 ++- .../unit_tests/integration/request_builder.py | 2 +- .../unit_tests/test_streams.py | 1 + .../integration_tests/test_acceptance.py | 6 +- .../source_s3/v4/legacy_config_transformer.py | 4 +- .../unit_tests/v4/test_stream_reader.py | 79 ++- .../integration_tests/integration_test.py | 6 +- .../integration_tests/state_migration.py | 2 +- .../source_salesforce/rate_limiting.py | 8 +- .../source-salesforce/unit_tests/api_test.py | 52 +- .../integration/test_bulk_stream.py | 94 ++- .../integration/test_rest_stream.py | 53 +- .../unit_tests/integration/test_source.py | 9 +- .../unit_tests/integration/utils.py | 13 +- .../salesforce_job_response_builder.py | 28 +- .../unit_tests/test_rate_limiting.py | 38 +- .../unit_tests/test_slice_generation.py | 4 +- .../unit_tests/integration/config_builder.py | 2 +- .../integration/test_events_stream.py | 16 +- .../integration/test_issues_stream.py | 10 +- .../source-sentry/unit_tests/test_streams.py | 20 +- .../integration_tests/integration_test.py | 14 +- .../source-shopify/source_shopify/auth.py | 1 - .../source_shopify/streams/streams.py | 1 - .../source-shopify/source_shopify/utils.py | 2 +- .../source-shopify/unit_tests/conftest.py | 19 +- .../unit_tests/graphql_bulk/test_job.py | 75 +- .../unit_tests/graphql_bulk/test_query.py | 209 ++++-- .../unit_tests/graphql_bulk/test_record.py | 2 +- .../unit_tests/integration/api/bulk.py | 71 +- .../integration/test_bulk_stream.py | 111 ++- .../unit_tests/test_deleted_events_stream.py | 4 +- .../unit_tests/test_graphql_products.py | 23 +- .../source-shopify/unit_tests/test_source.py | 46 +- .../source-shopify/unit_tests/unit_test.py | 5 +- .../source-slack/unit_tests/conftest.py | 39 +- .../unit_tests/test_components.py | 48 +- .../source-slack/unit_tests/test_source.py | 1 + .../source-slack/unit_tests/test_streams.py | 57 +- .../integration/test_payment_methods.py | 63 +- .../source-stripe/unit_tests/test_streams.py | 27 +- .../unit_tests/test_source.py | 8 +- .../unit_tests/conftest.py | 21 +- .../unit_tests/test_streams.py | 36 +- .../unit_tests/test_paginator.py | 35 +- .../semi_incremental_record_filter.py | 1 - .../unit_tests/integration/config_builder.py | 9 +- .../integration/test_reports_hourly.py | 94 +-- .../unit_tests/test_components.py | 172 ++--- .../unit_tests/test_source.py | 59 +- .../source-twilio/unit_tests/test_streams.py | 24 +- .../unit_tests/test_authenticator.py | 8 +- .../source-webflow/source_webflow/source.py | 1 - .../webflow_to_airbyte_mapping.py | 1 - .../unit_tests/test_custom_parsing.py | 8 +- .../source-xero/unit_tests/test_source.py | 1 + .../source-xero/unit_tests/test_streams.py | 14 +- .../components/id_incremental_cursor.py | 6 +- .../unit_tests/components/conftest.py | 29 +- .../components/test_bans_record_extractor.py | 4 +- .../components/test_id_incremental_cursor.py | 36 +- .../components/test_id_offset_pagination.py | 13 +- .../components/test_time_offset_pagination.py | 12 +- .../components/test_timestamp_based_cursor.py | 26 +- .../unit_tests/integrations/helpers.py | 28 +- .../unit_tests/integrations/test_groups.py | 4 +- .../integrations/test_post_comment_votes.py | 16 +- .../integrations/test_post_comments.py | 16 +- .../integrations/test_post_votes.py | 16 +- .../integrations/test_ticket_metrics.py | 26 +- .../zs_requests/base_request_builder.py | 7 +- .../post_comment_votes_request_builder.py | 4 +- .../unit_tests/unit_test.py | 141 ++-- 300 files changed, 4697 insertions(+), 4763 deletions(-) diff --git a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py index afe74946cb106..56f6389a8b346 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py @@ -10,19 +10,23 @@ DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO = Path("docs/contributing-to-airbyte/resources/qa-checks.md") + @pytest.fixture def airbyte_repo(): return git.Repo(search_parent_directories=True) + @pytest.mark.asyncio async def test_generated_qa_checks_documentation_is_up_to_date(airbyte_repo, tmp_path): # Arrange current_doc = (airbyte_repo.working_dir / DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO).read_text() newly_generated_doc_path = tmp_path / "qa-checks.md" - + # Act await CliRunner().invoke(generate_documentation, [str(tmp_path / "qa-checks.md")], catch_exceptions=False) # Assert suggested_command = f"connectors-qa generate-documentation {DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO}" - assert newly_generated_doc_path.read_text() == current_doc, f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." + assert ( + newly_generated_doc_path.read_text() == current_doc + ), f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes." diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py index ed4e8eebbfe96..f26d49671fdd5 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py @@ -42,7 +42,6 @@ def test_fail_when_icon_path_is_not_named_icon_svg(self, tmp_path, mocker): assert result.status == CheckStatus.FAILED assert result.message == "Icon file is not a SVG file" - def test_fail_when_icon_file_is_not_valid_svg(self, tmp_path, mocker): # Arrange connector = mocker.MagicMock() diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py index 959ba6b48311e..833f7c12f4fb7 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py @@ -204,7 +204,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat ab_internal_sl=300, language="python", connector_type="source", - documentation_file_path=tmp_path / "not_existing_documentation.md" + documentation_file_path=tmp_path / "not_existing_documentation.md", ) # Act @@ -217,11 +217,7 @@ def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_pat def test_fail_when_documentation_file_path_is_none(self, mocker): # Arrange connector = mocker.Mock( - technical_name="test-connector", - ab_internal_sl=300, - language="python", - connector_type="source", - documentation_file_path=None + technical_name="test-connector", ab_internal_sl=300, language="python", connector_type="source", documentation_file_path=None ) # Act @@ -263,34 +259,28 @@ def test_fail_when_documentation_file_has_missing_headers(self, connector_with_i assert "Actual Heading: 'For Airbyte Cloud:'. Expected Heading: 'Setup guide'" in result.message def test_fail_when_documentation_file_not_have_all_required_fields_in_prerequisites_section_content( - self, - connector_with_invalid_documentation + self, connector_with_invalid_documentation ): # Act - result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run( - connector_with_invalid_documentation - ) + result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Missing descriptions for required spec fields: github repositories" in result.message - def test_fail_when_documentation_file_has_invalid_source_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_source_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector GitHub section content does not follow standard template:" in result.message - assert "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." in result.message + assert ( + "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." + in result.message + ) - def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_invalid_documentation) @@ -299,10 +289,7 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_cont assert "Connector For Airbyte Cloud: section content does not follow standard template:" in result.message assert "+ 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account." in result.message - def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_invalid_documentation) @@ -311,23 +298,19 @@ def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_conte assert "Connector For Airbyte Open Source: section content does not follow standard template" in result.message assert "+ 1. Navigate to the Airbyte Open Source dashboard." in result.message - def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_invalid_documentation) # Assert assert result.status == CheckStatus.FAILED assert "Connector Supported sync modes section content does not follow standard template:" in result.message - assert ("+ The GitHub source connector supports the following" - " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):") in result.message + assert ( + "+ The GitHub source connector supports the following" + " [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):" + ) in result.message - def test_fail_when_documentation_file_has_invalid_tutorials_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_tutorials_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_invalid_documentation) @@ -336,10 +319,7 @@ def test_fail_when_documentation_file_has_invalid_tutorials_section_content( assert "Connector Tutorials section content does not follow standard template:" in result.message assert "+ Now that you have set up the GitHub source connector, check out the following GitHub tutorials:" in result.message - def test_fail_when_documentation_file_has_invalid_changelog_section_content( - self, - connector_with_invalid_documentation - ): + def test_fail_when_documentation_file_has_invalid_changelog_section_content(self, connector_with_invalid_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_invalid_documentation) @@ -356,10 +336,7 @@ def test_pass_when_documentation_file_has_correct_headers(self, connector_with_c assert result.status == CheckStatus.PASSED assert result.message == "Documentation guidelines are followed" - def test_pass_when_documentation_file_has_correct_prerequisites_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_prerequisites_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_correct_documentation) @@ -367,10 +344,7 @@ def test_pass_when_documentation_file_has_correct_prerequisites_section_content( assert result.status == CheckStatus.PASSED assert "All required fields from spec are present in the connector documentation" in result.message - def test_pass_when_documentation_file_has_correct_source_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_source_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSourceSectionContent()._run(connector_with_correct_documentation) @@ -378,10 +352,7 @@ def test_pass_when_documentation_file_has_correct_source_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_correct_documentation) @@ -389,10 +360,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_cont assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_correct_documentation) @@ -400,10 +368,7 @@ def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_conte assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_correct_documentation) @@ -411,10 +376,7 @@ def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_c assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_tutorials_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_tutorials_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckTutorialsSectionContent()._run(connector_with_correct_documentation) @@ -422,10 +384,7 @@ def test_pass_when_documentation_file_has_correct_tutorials_section_content( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_headers_order( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_headers_order(self, connector_with_correct_documentation): # Act result = documentation.CheckDocumentationHeadersOrder()._run(connector_with_correct_documentation) @@ -433,10 +392,7 @@ def test_pass_when_documentation_file_has_correct_headers_order( assert result.status == CheckStatus.PASSED assert "Documentation guidelines are followed" in result.message - def test_pass_when_documentation_file_has_correct_changelog_section_content( - self, - connector_with_correct_documentation - ): + def test_pass_when_documentation_file_has_correct_changelog_section_content(self, connector_with_correct_documentation): # Act result = documentation.CheckChangelogSectionContent()._run(connector_with_correct_documentation) diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py index b494c9d400741..56f98ae27081c 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py @@ -133,6 +133,7 @@ def test_pass_if_publish_to_pypi_is_disabled(self, mocker): assert result.status == CheckStatus.PASSED assert "PyPi publishing is declared" in result.message + class TestCheckConnectorLicense: def test_fail_when_license_is_missing(self, mocker): # Arrange diff --git a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py index c6c997b03fd69..c76057def50ad 100644 --- a/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py +++ b/airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py @@ -56,9 +56,7 @@ "connectorTestSuitesOptions": [ { "suite": testing.AcceptanceTestsEnabledCheck.test_suite_name, - "testSecrets": { - "testSecret": "test" - }, + "testSecrets": {"testSecret": "test"}, }, { "suite": "unit", @@ -71,10 +69,10 @@ OTHER_USAGE_VALUES = ["low", "none", "unknown", None, ""] DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES = [ - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, - METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS, + METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS, ] DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES = [ @@ -89,21 +87,9 @@ class TestAcceptanceTestsEnabledCheck: @pytest.mark.parametrize( "cases_to_test, usage_values_to_test, expected_result", [ - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - OTHER_USAGE_VALUES, - CheckStatus.SKIPPED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.PASSED - ), - ( - DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, - THRESHOLD_USAGE_VALUES, - CheckStatus.FAILED - ) + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, OTHER_USAGE_VALUES, CheckStatus.SKIPPED), + (DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.PASSED), + (DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.FAILED), ], ) def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases_to_test, usage_values_to_test, expected_result): @@ -115,11 +101,13 @@ def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases metadata=metadata_case, language=ConnectorLanguage.PYTHON, connector_type="source", - ab_internal_sl=100 + ab_internal_sl=100, ) # Act result = testing.AcceptanceTestsEnabledCheck().run(connector) # Assert - assert result.status == expected_result, f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" + assert ( + result.status == expected_result + ), f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}" diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py index 50a0209655cbb..303799e245ebb 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py @@ -13,5 +13,4 @@ class BaseBackend(ABC): """ @abstractmethod - def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: - ... + def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: ... diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py index a4d0b57c910a5..9863c561a8f30 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py @@ -123,7 +123,11 @@ def _get_filepaths_and_messages(self, message: AirbyteMessage) -> tuple[tuple[st stream_file_path_data_only = self.record_per_stream_directory / f"{sanitize_stream_name(stream_name)}_data_only.jsonl" self.record_per_stream_paths[stream_name] = stream_file_path self.record_per_stream_paths_data_only[stream_name] = stream_file_path_data_only - return (self.RELATIVE_RECORDS_PATH, str(stream_file_path), str(stream_file_path_data_only),), ( + return ( + self.RELATIVE_RECORDS_PATH, + str(stream_file_path), + str(stream_file_path_data_only), + ), ( message.json(sort_keys=True), message.json(sort_keys=True), json.dumps(message.record.data, sort_keys=True), diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py b/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py index eceb2686ed706..ea5070d4821b0 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/commons/proxy.py @@ -119,16 +119,18 @@ async def bind_container(self, container: dagger.Container) -> dagger.Container: python_version_minor_only = ".".join(python_version.split(".")[:-1]) requests_cert_path = f"/usr/local/lib/python{python_version_minor_only}/site-packages/certifi/cacert.pem" try: - return await ( - container.with_service_binding(self.hostname, await self.get_service()) - .with_mounted_cache("/mitmproxy_dir", self.mitmproxy_dir_cache) - .with_exec(["cp", cert_path_in_volume, requests_cert_path]) - .with_exec(["cp", cert_path_in_volume, ca_certificate_path]) - # The following command make the container use the proxy for all outgoing HTTP requests - .with_env_variable("REQUESTS_CA_BUNDLE", requests_cert_path) - .with_exec(["update-ca-certificates"]) - .with_env_variable("http_proxy", f"{self.hostname}:{self.PROXY_PORT}") - .with_env_variable("https_proxy", f"{self.hostname}:{self.PROXY_PORT}") + return ( + await ( + container.with_service_binding(self.hostname, await self.get_service()) + .with_mounted_cache("/mitmproxy_dir", self.mitmproxy_dir_cache) + .with_exec(["cp", cert_path_in_volume, requests_cert_path]) + .with_exec(["cp", cert_path_in_volume, ca_certificate_path]) + # The following command make the container use the proxy for all outgoing HTTP requests + .with_env_variable("REQUESTS_CA_BUNDLE", requests_cert_path) + .with_exec(["update-ca-certificates"]) + .with_env_variable("http_proxy", f"{self.hostname}:{self.PROXY_PORT}") + .with_env_variable("https_proxy", f"{self.hostname}:{self.PROXY_PORT}") + ) ) except dagger.DaggerError as e: # This is likely hapenning on Java connector images whose certificates location is different diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py index a9e2f6cd54fdb..48410830a0b0f 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/conftest.py @@ -108,11 +108,11 @@ def pytest_configure(config: Config) -> None: prompt_for_confirmation(user_email) track_usage( - "production-ci" - if config.stash[stash_keys.IS_PRODUCTION_CI] - else "local-ci" - if config.stash[stash_keys.RUN_IN_AIRBYTE_CI] - else user_email, + ( + "production-ci" + if config.stash[stash_keys.IS_PRODUCTION_CI] + else "local-ci" if config.stash[stash_keys.RUN_IN_AIRBYTE_CI] else user_email + ), vars(config.option), ) config.stash[stash_keys.AIRBYTE_API_KEY] = get_airbyte_api_key() diff --git a/airbyte-ci/connectors/live-tests/src/live_tests/report.py b/airbyte-ci/connectors/live-tests/src/live_tests/report.py index 255843f661434..0fa565eeb5b2d 100644 --- a/airbyte-ci/connectors/live-tests/src/live_tests/report.py +++ b/airbyte-ci/connectors/live-tests/src/live_tests/report.py @@ -114,9 +114,9 @@ def render(self) -> None: self.connection_objects.state if self.connection_objects.state else {}, indent=2, ), - configured_catalog=self.connection_objects.configured_catalog.json(indent=2) - if self.connection_objects.configured_catalog - else {}, + configured_catalog=( + self.connection_objects.configured_catalog.json(indent=2) if self.connection_objects.configured_catalog else {} + ), catalog=self.connection_objects.catalog.json(indent=2) if self.connection_objects.catalog else {}, message_count_per_type=self.get_message_count_per_type(), stream_coverage_metrics=self.get_stream_coverage_metrics(), diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_support_refreshes.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_support_refreshes.yaml index 4e66c6d592ffd..289b67cec2e38 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_support_refreshes.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_support_refreshes.yaml @@ -12,4 +12,4 @@ data: supportsRefreshes: 123 license: MIT tags: - - language:java \ No newline at end of file + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml index 3351eb5ddee57..f957533379fff 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml @@ -16,8 +16,8 @@ data: upgradeDeadline: 2023-08-22 message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." scopedImpact: - - scopeType: stream - impactedScopes: [] + - scopeType: stream + impactedScopes: [] tags: - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py index 4a821bb318705..47bfab2eaa0e1 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py @@ -19,6 +19,7 @@ PATCHED_VALIDATORS = [v for v in commands.PRE_UPLOAD_VALIDATORS if v not in NOT_TEST_VALIDATORS] + # TEST VALIDATE COMMAND def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path): runner = CliRunner() diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py index 9ce15092fcb7a..5021d6c37abe7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py @@ -10,9 +10,10 @@ @pytest.fixture def mock_spec_cache(): - with patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, patch( - "google.cloud.storage.Client.bucket" - ) as MockBucket: + with ( + patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, + patch("google.cloud.storage.Client.bucket") as MockBucket, + ): # Create stub mock client and bucket MockClient.return_value diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py index 54b0cd4b8e5e3..b4882b6c4f2d7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py @@ -51,7 +51,7 @@ async def _run(self, connector_to_discover: Container) -> StepResult: command.append("--skip-llm-relationships") erd_directory = self._build_erd_container(connector_directory, discovered_catalog).with_exec(command).directory("/source/erd") - await (erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory)))) + await erd_directory.export(str(_get_erd_folder(self.context.connector.code_directory))) return StepResult(step=self, status=StepStatus.SUCCESS, output=erd_directory) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 68f75275ad03e..8bcbb35cd1d35 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -133,9 +133,9 @@ def to_html(self) -> str: template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url - template_context[ - "icon_url" - ] = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + template_context["icon_url"] = ( + f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + ) return template.render(template_context) async def save_html_report(self) -> None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index afecc5fe00ee6..8276c07894a63 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -207,9 +207,11 @@ def __init__(self, context: ConnectorContext) -> None: internal_tools=[ MountPath(INTERNAL_TOOL_PATHS.CONNECTORS_QA.value), ], - secret_env_variables={"DOCKER_HUB_USERNAME": context.docker_hub_username, "DOCKER_HUB_PASSWORD": context.docker_hub_password} - if context.docker_hub_username and context.docker_hub_password - else None, + secret_env_variables=( + {"DOCKER_HUB_USERNAME": context.docker_hub_username, "DOCKER_HUB_PASSWORD": context.docker_hub_password} + if context.docker_hub_username and context.docker_hub_password + else None + ), command=["connectors-qa", "run", f"--name={technical_name}"], ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py index de33dfcb254a1..5b626bf0c9d3f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py @@ -36,9 +36,11 @@ def __init__(self, context: ConnectorContext) -> None: internal_tools=[ MountPath(INTERNAL_TOOL_PATHS.METADATA_SERVICE.value), ], - secret_env_variables={"DOCKER_HUB_USERNAME": context.docker_hub_username, "DOCKER_HUB_PASSWORD": context.docker_hub_password} - if context.docker_hub_username and context.docker_hub_password - else None, + secret_env_variables=( + {"DOCKER_HUB_USERNAME": context.docker_hub_username, "DOCKER_HUB_PASSWORD": context.docker_hub_password} + if context.docker_hub_username and context.docker_hub_password + else None + ), command=[ "metadata_service", "validate", diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py index 06276ef4f32d1..41033c83a9c86 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/client_container_runner.py @@ -55,7 +55,7 @@ async def _run_with_config(container: dagger.Container, command: List[str], conf async def _run(container: dagger.Container, command: List[str]) -> dagger.Container: - return await (container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command)) + return await container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec(command) async def get_client_container(dagger_client: dagger.Client, connector_path: Path, dockerfile_path: Path): diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py index 7435915090c75..e5546bf16415c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -14,7 +14,9 @@ "stream_configs, excluded_streams, expected_error_streams", [ pytest.param([{"name": "stream_with_primary_key", "primary_key": [["id"]]}], [], None, id="test_stream_with_primary_key_succeeds"), - pytest.param([{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails"), + pytest.param( + [{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails" + ), pytest.param([{"name": "report_stream"}], ["report_stream"], None, id="test_primary_key_excluded_from_test"), pytest.param( [ @@ -22,40 +24,55 @@ {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], [], ["himmel", "heiter"], id="test_multiple_streams_that_are_missing_primary_key"), + ], + [], + ["himmel", "heiter"], + id="test_multiple_streams_that_are_missing_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["himmel", "heiter"], None, id="test_multiple_streams_that_exclude_primary_key"), + ], + ["himmel", "heiter"], + None, + id="test_multiple_streams_that_exclude_primary_key", + ), pytest.param( [ {"name": "freiren", "primary_key": [["mage"]]}, {"name": "himmel"}, {"name": "eisen", "primary_key": [["warrior"]]}, {"name": "heiter"}, - ], ["heiter"], ["himmel"], id="test_multiple_streams_missing_primary_key_or_excluded"), + ], + ["heiter"], + ["himmel"], + id="test_multiple_streams_missing_primary_key_or_excluded", + ), ], ) async def test_streams_define_primary_key(mocker, stream_configs, excluded_streams, expected_error_streams): t = test_core.TestConnectorAttributes() - streams = [AirbyteStream.parse_obj({ - "name": stream_config.get("name"), - "json_schema": {}, - "default_cursor_field": ["updated_at"], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": stream_config.get("primary_key"), - }) for stream_config in stream_configs] + streams = [ + AirbyteStream.parse_obj( + { + "name": stream_config.get("name"), + "json_schema": {}, + "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": stream_config.get("primary_key"), + } + ) + for stream_config in stream_configs + ] streams_without_primary_key = [NoPrimaryKeyConfiguration(name=stream, bypass_reason="") for stream in excluded_streams] docker_runner_mock = mocker.MagicMock( - call_discover=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))] - ) + call_discover=mocker.AsyncMock(return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))]) ) if expected_error_streams: @@ -64,7 +81,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) streams_in_error_message = [stream_name for stream_name in expected_error_streams if stream_name in e.value.args[0]] assert streams_in_error_message == expected_error_streams @@ -73,7 +90,7 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea operational_certification_test=True, streams_without_primary_key=streams_without_primary_key, connector_config={}, - docker_runner=docker_runner_mock + docker_runner=docker_runner_mock, ) @@ -106,26 +123,22 @@ async def test_streams_define_primary_key(mocker, stream_configs, excluded_strea "Has `allowdHosts` but no `hosts`", "Has `hosts` but it's empty list", "Has non-empty `hosts`", - ] + ], ) async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_allowed_hosts( - operational_certification_test=True, - allowed_hosts_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, allowed_hosts_test=True, connector_metadata=metadata_yaml ) - + @pytest.mark.parametrize( "metadata_yaml, should_raise_assert_error, expected_error", @@ -156,22 +169,18 @@ async def test_certified_connector_has_allowed_hosts(metadata_yaml, should_raise "Has `suggestedStreams` but no `streams`", "Has `streams` but it's empty list", "Has non-empty `streams`", - ] + ], ) async def test_certified_connector_has_suggested_streams(metadata_yaml, should_raise_assert_error, expected_error) -> None: t = test_core.TestConnectorAttributes() - + if should_raise_assert_error: with pytest.raises(AssertionError) as e: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml ) assert expected_error in repr(e.value) else: await t.test_certified_connector_has_suggested_streams( - operational_certification_test=True, - suggested_streams_test=True, - connector_metadata=metadata_yaml - ) \ No newline at end of file + operational_certification_test=True, suggested_streams_test=True, connector_metadata=metadata_yaml + ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index 71799cfead083..bd9d05e2922ba 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -100,18 +100,11 @@ def test_discovery_uniquely_named_streams(): "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": ["null", "integer"] - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": ["null", "integer"]}}, + }, }, - True + True, ), ( { @@ -119,38 +112,22 @@ def test_discovery_uniquely_named_streams(): "type": ["null", "object"], "properties": { "amount": "integer", - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - True + True, ), ( { "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "amount": { - "type": ["null", "integer"] - }, - "amount_details": { - "type": ["null", "object"], - "properties": { - "atm_fee": { - "type": ["null", "integer"] - } - } - } - } + "amount": {"type": ["null", "integer"]}, + "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}}, + }, }, - False - ) + False, + ), ], ) def test_streams_have_valid_json_schemas(schema, should_fail): @@ -639,9 +616,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -658,9 +633,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -673,9 +646,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'object'}"), ), ( { @@ -688,9 +659,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -707,9 +676,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -722,9 +689,7 @@ def test_catalog_has_supported_data_types(discovered_catalog, expectation): }, ), }, - pytest.raises( - AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}" - ), + pytest.raises(AssertionError, match="Stream stream_1 contains primary key with forbidden type of {'array'}"), ), ( { @@ -883,7 +848,7 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca _DEFAULT_RECORD_CONFIG, [ {"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}, - {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]} + {"constant_field": "must equal", "fast_changing_field": [{"field": 2}]}, ], {"test_stream": [{"constant_field": "must equal", "fast_changing_field": [{"field": 1}]}]}, None, @@ -911,13 +876,13 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca ), # Expected is in actual but not in order (for case when exact_order=True) ( - {"type": "object"}, - {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, - ExpectedRecordsConfig(exact_order=True, path="foobar"), - [{"constant_field": "not in order"}, {"constant_field": "must equal"}], - {"test_stream": [{"constant_field": "must equal"}]}, - None, - does_not_raise(), + {"type": "object"}, + {"test_stream": [IgnoredFieldsConfiguration(name="fast_changing_field/*/field", bypass_reason="test")]}, + ExpectedRecordsConfig(exact_order=True, path="foobar"), + [{"constant_field": "not in order"}, {"constant_field": "must equal"}], + {"test_stream": [{"constant_field": "must equal"}]}, + None, + does_not_raise(), ), # Match by primary key ( @@ -985,12 +950,14 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream.parse_obj({ - "name": "test_stream", - "json_schema": schema, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": primary_key - }), + stream=AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": schema, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": primary_key, + } + ), sync_mode="full_refresh", destination_sync_mode="overwrite", ) @@ -998,7 +965,10 @@ async def test_read(mocker, schema, ignored_fields, expect_records_config, recor ) docker_runner_mock = mocker.MagicMock( call_read=mocker.AsyncMock( - return_value=[AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) for record in records] + return_value=[ + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) + for record in records + ] ) ) t = test_core.TestBasicRead() @@ -1954,8 +1924,8 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p ] ) stream = AirbyteStreamState( - stream_descriptor=StreamDescriptor(name='test_stream_0', namespace=None), - stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True) + stream_descriptor=StreamDescriptor(name="test_stream_0", namespace=None), + stream_state=AirbyteStateBlob(__ab_no_cursor_state_message=True), ) async_stream_output = [ AirbyteMessage( @@ -1989,7 +1959,7 @@ async def test_read_validate_async_output_state_messages(mocker, state_message_p stream_descriptor=StreamDescriptor(name="test_stream_0"), status=AirbyteStreamStatus.COMPLETE ), ), - ) + ), ] if not state_message_params: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py index 5a602e85a2e75..b0a3e4464ef9f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_documentation.py @@ -11,19 +11,19 @@ @pytest.mark.parametrize( "connector_spec, docs_path, should_fail", ( - # SUCCESS: required field from spec exists in Prerequisites section - ( - {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, - "data/docs/incorrect_not_all_structure.md", - False - ), - # FAIL: required field from spec does not exist in Prerequisites section - ( - {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, - "data/docs/incorrect_not_all_structure.md", - True - ) - ) + # SUCCESS: required field from spec exists in Prerequisites section + ( + {"required": ["start_date"], "properties": {"start_date": {"title": "Start Date"}}}, + "data/docs/incorrect_not_all_structure.md", + False, + ), + # FAIL: required field from spec does not exist in Prerequisites section + ( + {"required": ["access_token"], "properties": {"access_token": {"title": "Access Token"}}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + ), ) def test_documentation_prerequisites_section(connector_spec, docs_path, should_fail): t = _TestConnectorDocumentation() @@ -41,35 +41,35 @@ def test_documentation_prerequisites_section(connector_spec, docs_path, should_f @pytest.mark.parametrize( "metadata, docs_path, should_fail, failure", ( - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - "Missing headers:", - ), - # FAIL: Docs does not have required headers from standard template - ( - {"data": {"name": "Oracle Netsuite"}}, - "data/docs/with_not_required_steps.md", - True, - "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", - ), - # # SUCCESS: Docs follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - "", - ), - # Fail: Incorrect header order - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_header_order.md", - True, - "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", - ), - ) + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + "Missing headers:", + ), + # FAIL: Docs does not have required headers from standard template + ( + {"data": {"name": "Oracle Netsuite"}}, + "data/docs/with_not_required_steps.md", + True, + "Actual Heading: 'Create Oracle NetSuite account'. Possible correct heading", + ), + # # SUCCESS: Docs follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + "", + ), + # Fail: Incorrect header order + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_header_order.md", + True, + "Actual Heading: 'Prerequisites'. Expected Heading: 'GitHub'", + ), + ), ) def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, failure): t = _TestConnectorDocumentation() @@ -89,25 +89,25 @@ def test_docs_structure_is_correct(mocker, metadata, docs_path, should_fail, fai @pytest.mark.parametrize( "metadata, docs_path, should_fail", ( - # FAIL: Prerequisites section does not follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/incorrect_not_all_structure.md", - True, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct.md", - False, - ), - # SUCCESS: Section descriptions follow standard template - ( - {"data": {"name": "GitHub"}}, - "data/docs/correct_all_description_exist.md", - False, - ), - ) + # FAIL: Prerequisites section does not follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/incorrect_not_all_structure.md", + True, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct.md", + False, + ), + # SUCCESS: Section descriptions follow standard template + ( + {"data": {"name": "GitHub"}}, + "data/docs/correct_all_description_exist.md", + False, + ), + ), ) def test_docs_description(mocker, metadata, docs_path, should_fail): mocker.patch.object(conftest.pytest, "fail") @@ -140,7 +140,7 @@ def test_docs_description(mocker, metadata, docs_path, should_fail): "data/docs/correct.md", False, ), - ) + ), ) def test_docs_urls(docs_path, should_fail): t = _TestConnectorDocumentation() diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index 05724361dab07..3616709376cf2 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -56,7 +56,10 @@ def build_state_message(state: dict) -> AirbyteMessage: def build_per_stream_state_message( - descriptor: StreamDescriptor, stream_state: Optional[dict[str, Any]], data: Optional[dict[str, Any]] = None, source_stats: Optional[dict[str, Any]] = None + descriptor: StreamDescriptor, + stream_state: Optional[dict[str, Any]], + data: Optional[dict[str, Any]] = None, + source_stats: Optional[dict[str, Any]] = None, ) -> AirbyteMessage: if data is None: data = stream_state @@ -70,7 +73,7 @@ def build_per_stream_state_message( type=AirbyteStateType.STREAM, stream=AirbyteStreamState(stream_descriptor=descriptor, stream_state=stream_state_blob), sourceStats=AirbyteStateStats(**source_stats), - data=data + data=data, ), ) @@ -232,20 +235,40 @@ async def test_incremental_two_sequential_reads( [ {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, ], # Read after 2022-05-10. This is the second (last) subsequent read. [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-10"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, - ] + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-12"}, + "sourceStats": {"recordCount": 3.0}, + }, + ], ], IncrementalConfig(), does_not_raise(), @@ -258,9 +281,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 0.0}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_records_on_first_read_skips_stream", @@ -274,9 +295,7 @@ async def test_incremental_two_sequential_reads( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, ], - [ - [] - ], + [[]], IncrementalConfig(), does_not_raise(), id="test_incremental_no_states_on_first_read_skips_stream", @@ -293,13 +312,28 @@ async def test_incremental_two_sequential_reads( ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-08"}, + "sourceStats": {"recordCount": 2.0}, + }, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}, "sourceStats": {"recordCount": 2.0}}, + { + "type": Type.STATE, + "name": "test_stream", + "stream_state": {"date": "2022-05-13"}, + "sourceStats": {"recordCount": 2.0}, + }, ], ], IncrementalConfig(), @@ -372,20 +406,30 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse ) call_read_output_messages = [ - build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + ( + build_per_stream_state_message( + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), + ) + if record["type"] == Type.STATE + else build_record_message(record["name"], record["data"]) ) - if record["type"] == Type.STATE - else build_record_message(record["name"], record["data"]) for record in list(first_records) ] call_read_with_state_output_messages = [ [ - build_per_stream_state_message( - descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None), source_stats=record.get("sourceStats") + ( + build_per_stream_state_message( + descriptor=StreamDescriptor(name=record["name"]), + stream_state=record["stream_state"], + data=record.get("data", None), + source_stats=record.get("sourceStats"), + ) + if record["type"] == Type.STATE + else build_record_message(stream=record["name"], data=record["data"]) ) - if record["type"] == Type.STATE - else build_record_message(stream=record["name"], data=record["data"]) for record in state_records_group ] for state_records_group in list(subsequent_records) @@ -416,22 +460,20 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse [ {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {}, "sourceStats": {"recordCount": 0.0}}, ], [], [], - id="combine_three_duplicates_into_a_single_state_message" + id="combine_three_duplicates_into_a_single_state_message", ), pytest.param( [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}} - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, ], + [{"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}], [0.0], - id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal" + id="multiple_equal_states_with_different_sourceStats_considered_to_be_equal", ), pytest.param( [ @@ -443,27 +485,33 @@ async def test_per_stream_read_with_multiple_states(mocker, first_records, subse {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 0.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-08"}, "sourceStats": {"recordCount": 2.0}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-10"}, "sourceStats": {"recordCount": 7.0}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}} + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-12"}, "sourceStats": {"recordCount": 3.0}}, ], - [10.0, 3.0, 0.0] - ) + [10.0, 3.0, 0.0], + ), ], ) async def test_get_unique_state_messages(non_unique_states, expected_unique_states, expected_record_count_per_state): non_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in non_unique_states ] expected_unique_states = [ build_per_stream_state_message( - descriptor=StreamDescriptor(name=state["name"]), stream_state=state["stream_state"], data=state.get("data", None), source_stats=state.get("sourceStats") + descriptor=StreamDescriptor(name=state["name"]), + stream_state=state["stream_state"], + data=state.get("data", None), + source_stats=state.get("sourceStats"), ) for state in expected_unique_states ] @@ -471,7 +519,9 @@ async def test_get_unique_state_messages(non_unique_states, expected_unique_stat assert len(actual_unique_states) == len(expected_unique_states) if len(expected_unique_states): - for actual_state_data, expected_state, expected_record_count in zip(actual_unique_states, expected_unique_states, expected_record_count_per_state): + for actual_state_data, expected_state, expected_record_count in zip( + actual_unique_states, expected_unique_states, expected_record_count_per_state + ): actual_state, actual_record_count = actual_state_data assert actual_state == expected_state assert actual_record_count == expected_record_count @@ -517,7 +567,8 @@ async def test_config_skip_test(mocker): IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration())), [], {"type": "object", "properties": {"date": {"type": "str"}}}, - pytest.raises(AssertionError), id="Error because incremental stream should always emit state messages" + pytest.raises(AssertionError), + id="Error because incremental stream should always emit state messages", ), pytest.param( [ @@ -561,20 +612,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -594,25 +634,16 @@ async def test_config_skip_test(mocker): ), ) ], - IncrementalConfig(future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$"))), + IncrementalConfig( + future_state=FutureStateConfig(cursor_format=FutureStateCursorFormatConfiguration(format="^\\d{4}-\\d{2}-\\d{2}$")) + ), [ { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-10-12" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-10-12"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -637,20 +668,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": "2222-05-08T03:04:45.139-0700" - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": "2222-05-08T03:04:45.139-0700"}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": "str"}}}, @@ -676,20 +696,9 @@ async def test_config_skip_test(mocker): { "type": "STREAM", "stream": { - "stream_descriptor": { - "name": "test_stream" - }, - "stream_state": { - "states": [ - { - "partition": {}, - "cursor": { - "date": 10000000.0 - } - } - ] - } - } + "stream_descriptor": {"name": "test_stream"}, + "stream_state": {"states": [{"partition": {}, "cursor": {"date": 10000000.0}}]}, + }, } ], {"type": "object", "properties": {"date": {"type": ["int", "null"]}}}, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 4b4b1d456e756..45471756246c9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -681,7 +681,7 @@ def test_enum_usage(connector_spec, should_fail): }, ), "", - ) + ), ], ) def test_validate_oauth_flow(connector_spec, expected_error): @@ -698,227 +698,153 @@ def test_validate_oauth_flow(connector_spec, expected_error): [ # FAIL: OAuth is not default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, }, - ] - } - } + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "Oauth method should be a default option. Current default method is access_token." + }, + }, + }, + ), + "Oauth method should be a default option. Current default method is access_token.", ), # SUCCESS: Oauth is default ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": { - "type": "string" - }, - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - } - ] - } - } + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "string"}, + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + ], + }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": { - "domain": { - "type": "string", - "path_in_connector_config": ["api_url"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { "type": "string", "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } - ), "" + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + }, + }, + ), + "", ), # SUCCESS: no advancedAuth specified (ConnectorSpecification(connectionSpecification={}), ""), @@ -992,60 +918,60 @@ def test_validate_oauth_flow(connector_spec, expected_error): ), # SUCCESS: Skipped: no predicate key. ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "api_url": {"type": "object"}, - "credentials": { - "type": "object", - "properties": { - "auth_type": {"type": "string", "const": "oauth2.0"}, - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - "token_expiry_date": {"type": "string", "format": "date-time"}, - }, + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "api_url": {"type": "object"}, + "credentials": { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, }, }, }, - advanced_auth={ - "auth_flow_type": "oauth2.0", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, - }, - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, - "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"], - }, + }, + advanced_auth={ + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], }, }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, - "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, - }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, }, }, }, - ), - "Advanced Auth object does not have predicate_key, only one option to authenticate.", - ) - ] + }, + ), + "Advanced Auth object does not have predicate_key, only one option to authenticate.", + ), + ], ) def test_validate_auth_default_method(connector_spec, expected_error): t = _TestSpec() @@ -1106,69 +1032,66 @@ def test_additional_properties_is_true(connector_spec, expectation): ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string"}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": {"client_secret": {"type": "string"}, "access_token": {"type": "string", "airbyte_secret": True}}, + } + }, }, - True + True, ), ( { "type": "object", - "properties": {"credentials": {"type": "object", "properties": { - "client_secret": {"type": "string", "airbyte_secret": True}, - "access_token": {"type": "string", "airbyte_secret": True}}}} + "properties": { + "credentials": { + "type": "object", + "properties": { + "client_secret": {"type": "string", "airbyte_secret": True}, + "access_token": {"type": "string", "airbyte_secret": True}, + }, + } + }, }, - False + False, ), ( - { - "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - } - } - }, - { - "type": "object", - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - }, - "access_token": { - "type": "string" - }, - "token_expiry_date": { - "type": "string", - }, - "refresh_token": { - "type": "string", - } - } - }, - ] - } - } + { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "access_token"}, + "access_token": { + "type": "string", + }, + }, + }, + { + "type": "object", + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "token_expiry_date": { + "type": "string", + }, + "refresh_token": { + "type": "string", + }, + }, + }, + ], + } }, - True + }, + True, ), ({"type": "object", "properties": {"auth": {"oneOf": [{"api_token": {"type": "string"}}]}}}, True), ( @@ -1187,7 +1110,9 @@ def test_airbyte_secret(mocker, connector_spec, should_fail): t = _TestSpec() logger = mocker.Mock() t.test_secret_is_properly_marked( - {"connectionSpecification": connector_spec}, logger, ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret") + {"connectionSpecification": connector_spec}, + logger, + ("api_key", "api_token", "refresh_token", "jwt", "credentials", "access_token", "client_secret"), ) if should_fail: conftest.pytest.fail.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py index b9d1aac8ae3c9..a93353ee2277e 100644 --- a/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-astra/integration_tests/integration_test.py @@ -19,12 +19,11 @@ def test_check_valid_config(self): assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): - invalid_config = self.config + invalid_config = self.config invalid_config["embedding"]["openai_key"] = 123 - outcome = DestinationAstra().check( - logging.getLogger("airbyte"), invalid_config) + outcome = DestinationAstra().check(logging.getLogger("airbyte"), invalid_config) assert outcome.status == Status.FAILED def test_write(self): @@ -32,11 +31,7 @@ def test_write(self): embedder = create_from_config(db_config.embedding, db_config.processing) db_creds = db_config.indexing astra_client = AstraClient( - db_creds.astra_db_endpoint, - db_creds.astra_db_app_token, - db_creds.astra_db_keyspace, - embedder.embedding_dimensions, - "cosine" + db_creds.astra_db_endpoint, db_creds.astra_db_app_token, db_creds.astra_db_keyspace, embedder.embedding_dimensions, "cosine" ) astra_client.delete_documents(collection_name=db_creds.collection, filter={}) @@ -49,4 +44,3 @@ def test_write(self): outcome = list(DestinationAstra().write(self.config, catalog, [message1, message2])) assert astra_client.count_documents(db_creds.collection) == 2 - diff --git a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py index ebb1d41e230a0..b6bd3d5bb0fc6 100644 --- a/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py +++ b/airbyte-integrations/connectors/destination-astra/unit_tests/indexer_test.py @@ -142,7 +142,9 @@ def test_astra_pre_sync(): ("other_collection", None, 3, False, "mycollection collection does not exist."), ( ["mycollection"], - urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.")), + urllib3.exceptions.MaxRetryError( + None, "", reason=Exception("Failed to resolve environment, please check whether the credential is correct.") + ), 3, False, "Failed to resolve environment", @@ -157,12 +159,16 @@ def test_astra_check(collection_name, describe_throws, reported_dimensions, chec indexer.client.create_collection = MagicMock() indexer.client.find_collections = MagicMock() - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] if describe_throws: indexer.client.find_collections.side_effect = describe_throws else: - indexer.client.find_collections.return_value = [create_index_description(collection_name=collection_name, dimensions=reported_dimensions)] + indexer.client.find_collections.return_value = [ + create_index_description(collection_name=collection_name, dimensions=reported_dimensions) + ] result = indexer.check() if check_succeeds: diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py index 223312f5d05ba..805b891103e71 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/destination.py @@ -34,7 +34,6 @@ def _get_random_string(length: int) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py index 29f792fbc02e8..cc008357f407c 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/integration_tests/integration_test.py @@ -95,13 +95,13 @@ def test_check_invalid_aws_account_config(invalid_account_config: Mapping): def _state(stream: str, data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage( - type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_state=data, - stream_descriptor=StreamDescriptor(name=stream, namespace=None) - ) - )) + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState(stream_state=data, stream_descriptor=StreamDescriptor(name=stream, namespace=None)), + ), + ) def _record(stream: str, str_value: str, int_value: int, date_value: datetime) -> AirbyteMessage: diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py index b0926ffd3c659..8dda62704cc3c 100644 --- a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -42,7 +42,6 @@ def _init_indexer(self, config: ConfigModel): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py index f8da4b0984bde..0b0ff0ebf1de2 100644 --- a/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py +++ b/airbyte-integrations/connectors/destination-databend/destination_databend/destination.py @@ -23,7 +23,6 @@ class DestinationDatabend(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ TODO Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py index 2372f49736fbb..eb46c9c71630b 100644 --- a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py +++ b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py @@ -117,14 +117,14 @@ def test_connection(config: Dict[str, str], logger: MagicMock) -> None: @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_append( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ) -> None: catalog = ConfiguredAirbyteCatalog(streams=[configured_stream1, configured_stream2]) @@ -141,14 +141,14 @@ def test_sql_write_append( @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_overwrite( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ): # Overwrite triggers a delete configured_stream1.destination_sync_mode = DestinationSyncMode.overwrite diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py index d985da9706db0..5a8c6b23dd38b 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py @@ -32,9 +32,7 @@ from faker import Faker CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +43,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -102,6 +98,7 @@ def test_large_table_name() -> str: rand_string = "".join(random.choice(letters) for _ in range(10)) return f"airbyte_integration_{rand_string}" + @pytest.fixture def table_schema() -> str: schema = {"type": "object", "properties": {"column1": {"type": ["null", "string"]}}} @@ -110,7 +107,9 @@ def table_schema() -> str: @pytest.fixture def configured_catalogue( - test_table_name: str, test_large_table_name: str, table_schema: str, + test_table_name: str, + test_large_table_name: str, + table_schema: str, ) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( stream=AirbyteStream( @@ -159,9 +158,7 @@ def airbyte_message2(test_table_name: str): @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.mark.disable_autouse @@ -208,9 +205,7 @@ def test_write( if motherduck_api_key: duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: cursor = con.execute( "SELECT _airbyte_ab_id, _airbyte_emitted_at, _airbyte_data " @@ -222,21 +217,20 @@ def test_write( assert result[0][2] == json.dumps(airbyte_message1.record.data) assert result[1][2] == json.dumps(airbyte_message2.record.data) + def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, - data={"key1": fake.first_name() , "key2": fake.ssn()}, + data={"key1": fake.first_name(), "key2": fake.ssn()}, emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -250,28 +244,34 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab for i in range(n): if i != 0 and i % batch_size == 0: - yield AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)}) - ) + yield AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": str(i // batch_size)})) else: message = AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( stream=table_name, # Throw in empty nested objects and see how pyarrow deals with them. - data={"key1": fake.first_name(), - "key2": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "nested1": {} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": fake.ssn() if random.random()< 0.5 else random.randrange(1000,9999999999999), - "dictionary1":{} if random.random()< 0.1 else { - "key3": fake.first_name(), - "key4": "True" if random.random() < 0.5 else True - } - } - } - if random.random() < 0.9 else {}, - + data=( + { + "key1": fake.first_name(), + "key2": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "nested1": ( + {} + if random.random() < 0.1 + else { + "key3": fake.first_name(), + "key4": fake.ssn() if random.random() < 0.5 else random.randrange(1000, 9999999999999), + "dictionary1": ( + {} + if random.random() < 0.1 + else {"key3": fake.first_name(), "key4": "True" if random.random() < 0.5 else True} + ), + } + ), + } + if random.random() < 0.9 + else {} + ), emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -281,10 +281,15 @@ def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, tab TOTAL_RECORDS = 5_000 BATCH_WRITE_SIZE = 1000 + @pytest.mark.slow -@pytest.mark.parametrize("airbyte_message_generator,explanation", - [(_airbyte_messages, "Test writing a large number of simple json objects."), - (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema.")] ) +@pytest.mark.parametrize( + "airbyte_message_generator,explanation", + [ + (_airbyte_messages, "Test writing a large number of simple json objects."), + (_airbyte_messages_with_inconsistent_json_fields, "Test writing a large number of json messages with inconsistent schema."), + ], +) def test_large_number_of_writes( config: Dict[str, str], request, @@ -309,13 +314,8 @@ def test_large_number_of_writes( duckdb_config["motherduck_token"] = motherduck_api_key duckdb_config["custom_user_agent"] = "airbyte_intg_test" - con = duckdb.connect( - database=config.get("destination_path"), read_only=False, config=duckdb_config - ) + con = duckdb.connect(database=config.get("destination_path"), read_only=False, config=duckdb_config) with con: - cursor = con.execute( - "SELECT count(1) " - f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}" - ) + cursor = con.execute("SELECT count(1) " f"FROM {test_schema_name}._airbyte_raw_{test_large_table_name}") result = cursor.fetchall() assert result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py index 4f5aeefa09b7b..2843ebbbd097f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py +++ b/airbyte-integrations/connectors/destination-duckdb/unit_tests/destination_unit_tests.py @@ -15,6 +15,7 @@ def test_validated_sql_name() -> None: with pytest.raises(ValueError): validated_sql_name("invalid-name") + @patch("duckdb.connect") @patch("os.makedirs") def test_check(mock_connect, mock_makedirs) -> None: @@ -27,6 +28,7 @@ def test_check(mock_connect, mock_makedirs) -> None: result = destination.check(logger, config) assert result.status == Status.SUCCEEDED + @patch("duckdb.connect") @patch("os.makedirs") def test_check_failure(mock_connect, mock_makedirs) -> None: @@ -38,6 +40,7 @@ def test_check_failure(mock_connect, mock_makedirs) -> None: assert result.status == Status.FAILED assert "Test exception" in result.message + @patch("duckdb.connect") @patch("os.makedirs") def test_write(mock_connect, mock_makedirs) -> None: diff --git a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py index 46d960893fc81..332c16049e4e4 100644 --- a/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/destination_firebolt/destination.py @@ -75,7 +75,6 @@ class DestinationFirebolt(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py index d4252f97d5c13..be9cf5a66fea8 100644 --- a/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py +++ b/airbyte-integrations/connectors/destination-firebolt/unit_tests/test_firebolt_destination.py @@ -34,6 +34,7 @@ def config(request: Any) -> Dict[str, str]: } return args + @fixture() def legacy_config(): args = { @@ -45,6 +46,7 @@ def legacy_config(): } return args + @fixture def config_external_table() -> Dict[str, str]: args = { diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py index 2e1e4343ec55e..2a7dfe381df9e 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/destination.py @@ -40,7 +40,6 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. """ diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py index 29d27ae744fc7..1ebc23f87ea5b 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py @@ -39,7 +39,6 @@ def get_streams_from_catalog(catalog: ConfiguredAirbyteCatalog, limit: int = STR class ConnectionTest: - """ Performs connection test write operation to ensure the target spreadsheet is available for writing. Initiating the class itself, performs the connection test and stores the result in ConnectionTest.result property. diff --git a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/integration_test.py index 25c8d85623aa7..36da8f3b69703 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-google-sheets/integration_tests/integration_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + # fixture for the _customIntegrationTest test. def test_fixture(): assert True diff --git a/airbyte-integrations/connectors/destination-google-sheets/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-google-sheets/unit_tests/unit_test.py index 67990aabb278d..63394cbc1c9b6 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-google-sheets/unit_tests/unit_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + # fixture for the unit_tests def test_fixture(): assert True diff --git a/airbyte-integrations/connectors/destination-iceberg/finalize_build.py b/airbyte-integrations/connectors/destination-iceberg/finalize_build.py index 37062d29296a3..ebb0c716ad310 100644 --- a/airbyte-integrations/connectors/destination-iceberg/finalize_build.py +++ b/airbyte-integrations/connectors/destination-iceberg/finalize_build.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + # This function is async in case async operations are needed. async def finalize_build(connector_context, connector_container, *args, **kwargs): custom_java_opts = "--add-opens java.base/java.lang=ALL-UNNAMED \ diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py index cbbb5c90c0df8..0a23edbb4b624 100644 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py +++ b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py @@ -19,7 +19,6 @@ class DestinationKvdb(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py index 076bc70146854..59413f0221f7e 100644 --- a/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-motherduck/integration_tests/integration_test.py @@ -32,9 +32,7 @@ from faker import Faker CONFIG_PATH = "integration_tests/config.json" -SECRETS_CONFIG_PATH = ( - "secrets/config.json" # Should contain a valid MotherDuck API token -) +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token def pytest_generate_tests(metafunc): @@ -45,9 +43,7 @@ def pytest_generate_tests(metafunc): if Path(SECRETS_CONFIG_PATH).is_file(): configs.append("motherduck_config") else: - print( - f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}" - ) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") # for test_name in ["test_check_succeeds", "test_write"]: metafunc.parametrize("config", configs, indirect=True) @@ -89,9 +85,7 @@ def disable_destination_modification(monkeypatch, request): if "disable_autouse" in request.keywords: return else: - monkeypatch.setattr( - DestinationMotherDuck, "_get_destination_path", lambda _, x: x - ) + monkeypatch.setattr(DestinationMotherDuck, "_get_destination_path", lambda _, x: x) @pytest.fixture(scope="module") @@ -254,9 +248,7 @@ def airbyte_message2_update(airbyte_message2: AirbyteMessage, test_table_name: s @pytest.fixture def airbyte_message3(): - return AirbyteMessage( - type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"}) - ) + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) @pytest.fixture @@ -308,11 +300,7 @@ def _state(data: Dict[str, Any]) -> AirbyteMessage: @pytest.fixture() -def sql_processor( - configured_catalogue, - test_schema_name, - config: Dict[str, str] - ): +def sql_processor(configured_catalogue, test_schema_name, config: Dict[str, str]): destination = DestinationMotherDuck() path = config.get("destination_path", "md:") if CONFIG_MOTHERDUCK_API_KEY in config: @@ -320,7 +308,7 @@ def sql_processor( configured_catalog=configured_catalogue, schema_name=test_schema_name, db_path=path, - motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY] + motherduck_token=config[CONFIG_MOTHERDUCK_API_KEY], ) else: processor = destination._get_sql_processor( @@ -349,13 +337,7 @@ def test_write( generator = destination.write( config, configured_catalogue, - [ - airbyte_message1, - airbyte_message2, - airbyte_message3, - airbyte_message4, - airbyte_message5 - ], + [airbyte_message1, airbyte_message2, airbyte_message3, airbyte_message4, airbyte_message5], ) result = list(generator) @@ -407,9 +389,7 @@ def test_write_dupe( assert sql_result[1][1] == "777-54-0664" -def _airbyte_messages( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) @@ -431,9 +411,7 @@ def _airbyte_messages( yield message -def _airbyte_messages_with_inconsistent_json_fields( - n: int, batch_size: int, table_name: str -) -> Generator[AirbyteMessage, None, None]: +def _airbyte_messages_with_inconsistent_json_fields(n: int, batch_size: int, table_name: str) -> Generator[AirbyteMessage, None, None]: fake = Faker() Faker.seed(0) random.seed(0) @@ -453,25 +431,19 @@ def _airbyte_messages_with_inconsistent_json_fields( data=( { "key1": fake.unique.name(), - "key2": str(fake.ssn()) - if random.random() < 0.5 - else str(random.randrange(1000, 9999999999999)), + "key2": str(fake.ssn()) if random.random() < 0.5 else str(random.randrange(1000, 9999999999999)), "nested1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": str(fake.ssn()) - if random.random() < 0.5 - else random.randrange(1000, 9999999999999), + "key4": str(fake.ssn()) if random.random() < 0.5 else random.randrange(1000, 9999999999999), "dictionary1": ( {} if random.random() < 0.1 else { "key3": fake.first_name(), - "key4": "True" - if random.random() < 0.5 - else True, + "key4": "True" if random.random() < 0.5 else True, } ), } @@ -517,16 +489,11 @@ def test_large_number_of_writes( generator = destination.write( config, configured_catalogue, - airbyte_message_generator( - TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name - ), + airbyte_message_generator(TOTAL_RECORDS, BATCH_WRITE_SIZE, test_large_table_name), ) result = list(generator) assert len(result) == TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) - sql_result = sql_processor._execute_sql( - "SELECT count(1) " - f"FROM {test_schema_name}.{test_large_table_name}" - ) + sql_result = sql_processor._execute_sql("SELECT count(1) " f"FROM {test_schema_name}.{test_large_table_name}") assert sql_result[0][0] == TOTAL_RECORDS - TOTAL_RECORDS // (BATCH_WRITE_SIZE + 1) diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py index 67e610934d435..3ad3879c036bd 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/catalog/catalog_providers.py @@ -10,9 +10,10 @@ from typing import TYPE_CHECKING, Any, final -from airbyte import exceptions as exc from airbyte_cdk.models import DestinationSyncMode +from airbyte import exceptions as exc + if TYPE_CHECKING: from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py index 74bad59eca548..2dbf0c32b0913 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/destinations/record_processor.py @@ -14,18 +14,20 @@ from collections import defaultdict from typing import TYPE_CHECKING, cast, final -from airbyte import exceptions as exc -from airbyte.strategies import WriteStrategy from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, Type from destination_pgvector.common.state.state_writers import StateWriterBase, StdOutStateWriter +from airbyte import exceptions as exc +from airbyte.strategies import WriteStrategy + if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from airbyte._batch_handles import BatchHandle from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.common.state.state_writers import StateWriterBase + from airbyte._batch_handles import BatchHandle + class AirbyteMessageParsingError(Exception): """Raised when an Airbyte message is invalid or cannot be parsed.""" diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py index a297a01948469..b3c17d3944936 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/common/sql/sql_processor.py @@ -14,12 +14,6 @@ import pandas as pd import sqlalchemy import ulid -from airbyte import exceptions as exc -from airbyte._util.name_normalizers import LowerCaseNormalizer -from airbyte.constants import AB_EXTRACTED_AT_COLUMN, AB_META_COLUMN, AB_RAW_ID_COLUMN, DEBUG_MODE -from airbyte.progress import progress -from airbyte.strategies import WriteStrategy -from airbyte.types import SQLTypeConverter from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode from destination_pgvector.common.destinations.record_processor import RecordProcessorBase from destination_pgvector.common.state.state_writers import StdOutStateWriter @@ -28,12 +22,16 @@ from sqlalchemy import Column, Table, and_, create_engine, insert, null, select, text, update from sqlalchemy.sql.elements import TextClause +from airbyte import exceptions as exc +from airbyte._util.name_normalizers import LowerCaseNormalizer +from airbyte.constants import AB_EXTRACTED_AT_COLUMN, AB_META_COLUMN, AB_RAW_ID_COLUMN, DEBUG_MODE +from airbyte.progress import progress +from airbyte.strategies import WriteStrategy +from airbyte.types import SQLTypeConverter + if TYPE_CHECKING: from collections.abc import Generator - from airbyte._batch_handles import BatchHandle - from airbyte._processors.file.base import FileWriterBase - from airbyte.secrets.base import SecretString from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.common.state.state_writers import StateWriterBase @@ -43,6 +41,10 @@ from sqlalchemy.sql.base import Executable from sqlalchemy.sql.type_api import TypeEngine + from airbyte._batch_handles import BatchHandle + from airbyte._processors.file.base import FileWriterBase + from airbyte.secrets.base import SecretString + class RecordDedupeMode(enum.Enum): APPEND = "append" diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py index 32ae5291efdfa..72346ee05a979 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/destination.py @@ -8,8 +8,6 @@ from pathlib import Path from typing import Any, Iterable, Mapping, Optional -from airbyte.secrets import SecretString -from airbyte.strategies import WriteStrategy from airbyte_cdk.destinations import Destination from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -23,6 +21,9 @@ from destination_pgvector.common.catalog.catalog_providers import CatalogProvider from destination_pgvector.config import ConfigModel +from airbyte.secrets import SecretString +from airbyte.strategies import WriteStrategy + BATCH_SIZE = 150 diff --git a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py index 5757951e05d14..a444d0aa5367b 100644 --- a/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py +++ b/airbyte-integrations/connectors/destination-pgvector/destination_pgvector/pgvector_processor.py @@ -10,8 +10,6 @@ import dpath import sqlalchemy -from airbyte._processors.file.jsonl import JsonlWriter -from airbyte.secrets import SecretString from airbyte_cdk.destinations.vector_db_based import embedder from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor as DocumentSplitter from airbyte_cdk.destinations.vector_db_based.document_processor import ProcessingConfigModel as DocumentSplitterConfig @@ -23,6 +21,9 @@ from pgvector.sqlalchemy import Vector from typing_extensions import Protocol +from airbyte._processors.file.jsonl import JsonlWriter +from airbyte.secrets import SecretString + class PostgresConfig(SqlConfig): """Configuration for the Postgres cache. diff --git a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py index def5bb2988018..21dd3df8b7933 100644 --- a/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/integration_tests/integration_test.py @@ -331,7 +331,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationPGVector() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -358,4 +358,3 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["document_content"] == "Dogs are" assert third_written_record["document_id"] == "Stream_mystream_Key_0" assert third_written_record["document_content"] == "number 0" - diff --git a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py index 0b671c9ff1aaa..8e128db3595cc 100644 --- a/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-pgvector/unit_tests/destination_test.py @@ -6,11 +6,12 @@ import unittest from unittest.mock import MagicMock, Mock, patch -from airbyte.strategies import WriteStrategy from airbyte_cdk.models import ConnectorSpecification, Status from destination_pgvector.config import ConfigModel from destination_pgvector.destination import DestinationPGVector +from airbyte.strategies import WriteStrategy + class TestDestinationPGVector(unittest.TestCase): def setUp(self): diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py index 46215e878464d..aed6d4f4b6812 100644 --- a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py @@ -35,14 +35,14 @@ def _init_pinecone(self): self.pinecone_index = self.pc.Index(self.config["indexing"]["index"]) self.pc_rest = PineconeREST(api_key=self.config["indexing"]["pinecone_key"]) self.pinecone_index_rest = self.pc_rest.Index(name=self.config["indexing"]["index"]) - + def _wait(self): - print("Waiting for Pinecone...", end='', flush=True) + print("Waiting for Pinecone...", end="", flush=True) for i in range(15): time.sleep(1) - print(".", end='', flush=True) + print(".", end="", flush=True) print() # Move to the next line after the loop - + def setUp(self): with open("secrets/config.json", "r") as f: self.config = json.loads(f.read()) @@ -50,28 +50,28 @@ def setUp(self): def tearDown(self): self._wait() - # make sure pinecone is initialized correctly before cleaning up + # make sure pinecone is initialized correctly before cleaning up self._init_pinecone() try: self.pinecone_index.delete(delete_all=True) except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in default namespace. No data in the index/namespace.") try: self.pinecone_index.delete(delete_all=True, namespace="ns1") except PineconeException as e: if "Namespace not found" not in str(e): - raise(e) - else : + raise (e) + else: print("Nothing to delete in ns1 namespace. No data in the index/namespace.") def test_integration_test_flag_is_set(self): assert "PYTEST_CURRENT_TEST" in os.environ def test_check_valid_config(self): - outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) + outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(self): @@ -88,7 +88,7 @@ def test_check_invalid_config(self): }, }, ) - + assert outcome.status == Status.FAILED def test_write(self): @@ -99,21 +99,20 @@ def test_write(self): # initial sync destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - - - self._wait() + + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 # incrementalally update a doc incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) - - self._wait() - + + self._wait() + result = self.pinecone_index.query( vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_ab_record_id": "mystream_2"}, include_metadata=True ) - + assert len(result.matches) == 1 assert ( result.matches[0].metadata["text"] == "str_col: Cats are nice" @@ -135,19 +134,21 @@ def test_write_with_namespace(self): destination = DestinationPinecone() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - self._wait() + self._wait() assert self.pinecone_index.describe_index_stats().total_vector_count == 5 - def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}} + stream_schema = { + "type": "object", + "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}, "random_col": {"type": "integer"}}, + } overwrite_stream = ConfiguredAirbyteStream( stream=AirbyteStream( - name="mystream", + name="mystream", namespace="ns1", - json_schema=stream_schema, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + json_schema=stream_schema, + supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], ), primary_key=[["int_col"]], sync_mode=SyncMode.incremental, @@ -155,14 +156,9 @@ def _get_configured_catalog_with_namespace(self, destination_mode: DestinationSy ) return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) - + def _record_with_namespace(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, - namespace="ns1", - data={"str_col": str_value, "int_col": int_value}, - emitted_at=0) + type=Type.RECORD, + record=AirbyteRecordMessage(stream=stream, namespace="ns1", data={"str_col": str_value, "int_col": int_value}, emitted_at=0), ) - - - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py index 5d4b714a79024..4f29016140a3b 100644 --- a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py @@ -18,12 +18,12 @@ def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") - with patch.object(PineconeGRPC, 'Index') as mock_index: + with patch.object(PineconeGRPC, "Index") as mock_index: indexer = PineconeIndexer(config, 3) - + indexer.pc.list_indexes = MagicMock() indexer.pc.list_indexes.return_value.indexes = create_mock_list_indexes() - + indexer.pc.describe_index = MagicMock() if side_effect: indexer.pc.describe_index.side_effect = side_effect @@ -31,6 +31,7 @@ def create_pinecone_indexer(embedding_dimensions=3, side_effect=None): indexer.pc.describe_index.return_value = create_index_description(dimensions=embedding_dimensions) return indexer + def create_index_description(dimensions=3, pod_type="p1"): return IndexDescription( name="", @@ -41,18 +42,21 @@ def create_index_description(dimensions=3, pod_type="p1"): status=None, ) + def create_mock_list_indexes(): return [{"name": "myindex"}, {"name": "myindex2"}] + @pytest.fixture(scope="module", autouse=True) def mock_describe_index(): with patch("pinecone.describe_index") as mock: mock.return_value = create_index_description() yield mock + @pytest.fixture(scope="module", autouse=True) def mock_determine_spec_type(): - with patch.object(PineconeIndexer, 'determine_spec_type') as mock: + with patch.object(PineconeIndexer, "determine_spec_type") as mock: mock.return_value = "pod" yield mock @@ -77,7 +81,7 @@ def test_get_source_tag_with_pytest(): @patch.dict("os.environ", {"RUN_IN_AIRBYTE_CI": "Value does not matter"}) def test_get_source_tag_with_ci(): - # CI and pytest is running + # CI and pytest is running indexer = create_pinecone_indexer() assert indexer.get_source_tag() == "airbyte_test" @@ -143,6 +147,7 @@ def test_pinecone_index_upsert_and_delete_starter(mock_describe_index, mock_dete namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "pod" @@ -160,9 +165,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(filter={'_ab_record_id': {'$in': ['delete_id1', 'delete_id2']}}, namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}}, namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -173,6 +176,7 @@ def test_pinecone_index_upsert_and_delete_pod(mock_describe_index, mock_determin namespace="ns1", ) + def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_determine_spec_type): indexer = create_pinecone_indexer() indexer._pod_type = "serverless" @@ -190,9 +194,7 @@ def test_pinecone_index_upsert_and_delete_serverless(mock_describe_index, mock_d "some_stream", ) indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") - indexer.pinecone_index.delete.assert_has_calls( - [call(ids=['delete_id1', 'delete_id2'], namespace='ns1')] - ) + indexer.pinecone_index.delete.assert_has_calls([call(ids=["delete_id1", "delete_id2"], namespace="ns1")]) indexer.pinecone_index.upsert.assert_called_with( vectors=( (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), @@ -311,13 +313,12 @@ def test_pinecone_pre_sync_starter(mock_describe_index, mock_determine_spec_type ("myindex", None, 3, True, None), ("other_index", None, 3, False, "Index other_index does not exist in environment"), ( - "myindex", + "myindex", urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve 'controller.myenv.pinecone.io'")), 3, False, "Failed to resolve environment", - - ), + ), ("myindex", exceptions.UnauthorizedException(http_resp=urllib3.HTTPResponse(body="No entry!")), 3, False, "No entry!"), ("myindex", None, 4, False, "Make sure embedding and indexing configurations match."), ("myindex", Exception("describe failed"), 3, False, "describe failed"), diff --git a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py index a08452ebc2bd6..87a07ad189c24 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py +++ b/airbyte-integrations/connectors/destination-sftp-json/destination_sftp_json/destination.py @@ -20,7 +20,6 @@ def write( configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage], ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/catalog/catalog_providers.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/catalog/catalog_providers.py index 76adfe1a86a7e..8f8f267f791e7 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/catalog/catalog_providers.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/catalog/catalog_providers.py @@ -60,17 +60,13 @@ def get_configured_stream_info( ) matching_streams: list[ConfiguredAirbyteStream] = [ - stream - for stream in self.configured_catalog.streams - if stream.stream.name == stream_name + stream for stream in self.configured_catalog.streams if stream.stream.name == stream_name ] if not matching_streams: raise exc.AirbyteStreamNotFoundError( stream_name=stream_name, context={ - "available_streams": [ - stream.stream.name for stream in self.configured_catalog.streams - ], + "available_streams": [stream.stream.name for stream in self.configured_catalog.streams], }, ) diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/destinations/record_processor.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/destinations/record_processor.py index b40d8c18a1bbf..df7ad601eab60 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/destinations/record_processor.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/destinations/record_processor.py @@ -236,9 +236,7 @@ def process_airbyte_messages( stream_name = record_msg.stream if stream_name not in stream_schemas: - stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema( - stream_name=stream_name - ) + stream_schemas[stream_name] = self.catalog_provider.get_stream_json_schema(stream_name=stream_name) self.process_record_message( record_msg, diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py index 2867def1a4a99..54ff8172f3b20 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/common/sql/sql_processor.py @@ -121,9 +121,7 @@ def get_vendor_client(self) -> object: Raises `NotImplementedError` if a custom vendor client is not defined. """ - raise NotImplementedError( - f"The type '{type(self).__name__}' does not define a custom client." - ) + raise NotImplementedError(f"The type '{type(self).__name__}' does not define a custom client.") class SqlProcessorBase(RecordProcessorBase): @@ -287,9 +285,7 @@ def _get_table_by_name( query. To ignore the cache and force a refresh, set 'force_refresh' to True. """ if force_refresh and shallow_okay: - raise exc.PyAirbyteInternalError( - message="Cannot force refresh and use shallow query at the same time." - ) + raise exc.PyAirbyteInternalError(message="Cannot force refresh and use shallow query at the same time.") if force_refresh and table_name in self._cached_table_definitions: self._invalidate_table_cache(table_name) @@ -330,9 +326,7 @@ def _ensure_schema_exists( if DEBUG_MODE: found_schemas = self._get_schemas_list() - assert ( - schema_name in found_schemas - ), f"Schema {schema_name} was not created. Found: {found_schemas}" + assert schema_name in found_schemas, f"Schema {schema_name} was not created. Found: {found_schemas}" def _quote_identifier(self, identifier: str) -> str: """Return the given identifier, quoted.""" @@ -391,8 +385,7 @@ def _get_schemas_list( return [ found_schema.split(".")[-1].strip('"') for found_schema in found_schemas - if "." not in found_schema - or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) + if "." not in found_schema or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) ] def _ensure_final_table_exists( @@ -549,9 +542,7 @@ def finalizing_batches( Returns a mapping of batch IDs to batch handles, for those processed batches. """ batches_to_finalize: list[BatchHandle] = self.file_writer.get_pending_batches(stream_name) - state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[ - stream_name - ].copy() + state_messages_to_finalize: list[AirbyteStateMessage] = self._pending_state_messages[stream_name].copy() self._pending_state_messages[stream_name].clear() progress.log_batches_finalizing(stream_name, len(batches_to_finalize)) @@ -610,9 +601,7 @@ def _write_files_to_new_table( for file_path in files: dataframe = pd.read_json(file_path, lines=True) - sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions( - stream_name - ) + sql_column_definitions: dict[str, TypeEngine] = self._get_sql_column_definitions(stream_name) # Remove fields that are not in the schema for col_name in dataframe.columns: @@ -650,10 +639,7 @@ def _add_column_to_table( ) -> None: """Add a column to the given table.""" self._execute_sql( - text( - f"ALTER TABLE {self._fully_qualified(table.name)} " - f"ADD COLUMN {column_name} {column_type}" - ), + text(f"ALTER TABLE {self._fully_qualified(table.name)} " f"ADD COLUMN {column_name} {column_type}"), ) def _add_missing_columns_to_table( @@ -711,9 +697,7 @@ def _write_temp_table_to_final_table( ) if write_strategy == WriteStrategy.AUTO: - configured_destination_sync_mode: DestinationSyncMode = ( - self.catalog_provider.get_destination_sync_mode(stream_name) - ) + configured_destination_sync_mode: DestinationSyncMode = self.catalog_provider.get_destination_sync_mode(stream_name) if configured_destination_sync_mode == DestinationSyncMode.overwrite: write_strategy = WriteStrategy.REPLACE elif configured_destination_sync_mode == DestinationSyncMode.append: @@ -838,12 +822,13 @@ def _swap_temp_table_with_final_table( _ = stream_name deletion_name = f"{final_table_name}_deleteme" - commands = "\n".join([ - f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", - f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " - f"TO {final_table_name};", - f"DROP TABLE {self._fully_qualified(deletion_name)};", - ]) + commands = "\n".join( + [ + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME " f"TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(temp_table_name)} RENAME " f"TO {final_table_name};", + f"DROP TABLE {self._fully_qualified(deletion_name)};", + ] + ) self._execute_sql(commands) def _merge_temp_table_to_final_table( @@ -917,23 +902,16 @@ def _emulated_merge_temp_table_to_final_table( temp_table = self._get_table_by_name(temp_table_name) pk_columns = self._get_primary_keys(stream_name) - columns_to_update: set[str] = self._get_sql_column_definitions( - stream_name=stream_name - ).keys() - set(pk_columns) + columns_to_update: set[str] = self._get_sql_column_definitions(stream_name=stream_name).keys() - set(pk_columns) # Create a dictionary mapping columns in users_final to users_stage for updating update_values = { - self._get_column_by_name(final_table, column): ( - self._get_column_by_name(temp_table, column) - ) - for column in columns_to_update + self._get_column_by_name(final_table, column): (self._get_column_by_name(temp_table, column)) for column in columns_to_update } # Craft the WHERE clause for composite primary keys join_conditions = [ - self._get_column_by_name(final_table, pk_column) - == self._get_column_by_name(temp_table, pk_column) - for pk_column in pk_columns + self._get_column_by_name(final_table, pk_column) == self._get_column_by_name(temp_table, pk_column) for pk_column in pk_columns ] join_clause = and_(*join_conditions) @@ -948,9 +926,7 @@ def _emulated_merge_temp_table_to_final_table( where_not_exists_clause = self._get_column_by_name(final_table, pk_columns[0]) == null() # Select records from temp_table that are not in final_table - select_new_records_stmt = ( - select([temp_table]).select_from(joined_table).where(where_not_exists_clause) - ) + select_new_records_stmt = select([temp_table]).select_from(joined_table).where(where_not_exists_clause) # Craft the INSERT statement using the select statement insert_new_records_stmt = insert(final_table).from_select( diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/cortex_processor.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/cortex_processor.py index 10404de0d002a..300c2aacba459 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/cortex_processor.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/cortex_processor.py @@ -207,18 +207,15 @@ def path_str(path: Path) -> str: query = f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};" self._execute_sql(query) - columns_list = [ - self._quote_identifier(c) - for c in list(self._get_sql_column_definitions(stream_name).keys()) - ] + columns_list = [self._quote_identifier(c) for c in list(self._get_sql_column_definitions(stream_name).keys())] files_list = ", ".join([f"'{f.name}'" for f in files]) columns_list_str: str = indent("\n, ".join(columns_list), " " * 12) # following block is different from SnowflakeSqlProcessor vector_suffix = f"::Vector(Float, {self.embedding_dimensions})" - variant_cols_str: str = ("\n" + " " * 21 + ", ").join([ - f"$1:{col}{vector_suffix if 'embedding' in col else ''}" for col in columns_list - ]) + variant_cols_str: str = ("\n" + " " * 21 + ", ").join( + [f"$1:{col}{vector_suffix if 'embedding' in col else ''}" for col in columns_list] + ) if self.sql_config.cortex_embedding_model: # Currently always false # WARNING: This is untested and may not work as expected. variant_cols_str += f"snowflake.cortex.embed('{self.sql_config.cortex_embedding_model}', $1:{DOCUMENT_CONTENT_COLUMN})" @@ -273,9 +270,7 @@ def _emulated_merge_temp_table_to_final_table( So instead of using UPDATE and then INSERT, we will DELETE all rows for included primary keys and then call the append implementation to insert new rows. """ - columns_list: list[str] = list( - self._get_sql_column_definitions(stream_name=stream_name).keys() - ) + columns_list: list[str] = list(self._get_sql_column_definitions(stream_name=stream_name).keys()) delete_statement = dedent( f""" diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/destination.py b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/destination.py index 619519fa01459..e1af7afc95fda 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/destination.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/destination_snowflake_cortex/destination.py @@ -30,9 +30,7 @@ class DestinationSnowflakeCortex(Destination): sql_processor: cortex_processor.SnowflakeCortexSqlProcessor - def _init_sql_processor( - self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None - ): + def _init_sql_processor(self, config: ConfigModel, configured_catalog: Optional[ConfiguredAirbyteCatalog] = None): self.sql_processor = cortex_processor.SnowflakeCortexSqlProcessor( sql_config=cortex_processor.SnowflakeCortexConfig( host=config.indexing.host, @@ -73,9 +71,7 @@ def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionS self.sql_processor.sql_config.connect() return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: - return AirbyteConnectionStatus( - status=Status.FAILED, message=f"An exception occurred: {repr(e)}" - ) + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: return ConnectorSpecification( diff --git a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py index 864c227d28683..b970480b34b2c 100644 --- a/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-snowflake-cortex/integration_tests/integration_test.py @@ -337,7 +337,7 @@ def test_write_fidelity_with_chunk_size_5(self): for i in range(1) ] - # initial sync with replace + # initial sync with replace destination = DestinationSnowflakeCortex() list(destination.write(self.config, catalog, [*records, first_state_message])) assert self._get_record_count("mystream") == 3 @@ -364,8 +364,6 @@ def test_write_fidelity_with_chunk_size_5(self): assert second_written_record["DOCUMENT_CONTENT"] == '"Dogs are"' assert third_written_record["DOCUMENT_ID"] == "Stream_mystream_Key_0" assert third_written_record["DOCUMENT_CONTENT"] == '"number 0"' - - """ Following tests are not code specific, but are useful to confirm that the Cortex functions are available and behaving as expcected @@ -391,20 +389,24 @@ def test_get_embeddings_using_cortex(self): cur = conn.cursor() document_content_list = ["dogs are number 1", "dogs are number 2", "cats are nummber 1"] - cur.execute(""" + cur.execute( + """ CREATE TEMPORARY TABLE temp_document_content ( document_content STRING ) - """) + """ + ) cur.executemany( "INSERT INTO temp_document_content (document_content) VALUES (%s)", document_content_list, ) - cur.execute(""" + cur.execute( + """ SELECT snowflake.cortex.embed_text('e5-base-v2', document_content) AS embedding FROM temp_document_content - """) + """ + ) processed_data = cur.fetchall() self.assertTrue(processed_data, "No data found in the database") cur.execute("DROP TABLE temp_document_content") diff --git a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py index f8049536e7b42..fb2265fe4a134 100644 --- a/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py +++ b/airbyte-integrations/connectors/destination-sqlite/destination_sqlite/destination.py @@ -37,7 +37,6 @@ def _get_destination_path(destination_path: str) -> str: def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index 54e85d5512b74..b53b7dd91a024 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -35,6 +35,6 @@ def flush(self): for stream, data in self.write_buffer: grouped_by_stream[stream].append(data) - for (stream, data) in grouped_by_stream.items(): + for stream, data in grouped_by_stream.items(): self.client.collections[stream].documents.import_(data) self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py index b324865d36ba6..3c60ed65b96cf 100644 --- a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py @@ -24,7 +24,6 @@ class DestinationVectara(Destination): def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. diff --git a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py index 052006303d859..bf5104a684536 100644 --- a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py @@ -45,6 +45,7 @@ def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) ) + def _clean(self): self._client.delete_doc_by_metadata(metadata_field_name="_ab_stream", metadata_field_values=["None_mystream"]) diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py index 1930c55800b8a..91a0d2ce69114 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/conftest.py @@ -12,7 +12,7 @@ def config_pass(): "metrics": ["installs", "network_installs", "network_cost", "network_ecpi"], "dimensions": ["app", "partner_name", "campaign", "campaign_id_network", "campaign_network"], "additional_metrics": [], - "until_today": True + "until_today": True, } @@ -31,11 +31,7 @@ def mock_report_response(): return { "rows": [ { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,14 +40,9 @@ def mock_report_response(): "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } ], - "totals": { - "installs": 10.0, - "network_installs": 0.0, - "network_cost": 0.0, - "network_ecpi": 0.0 - }, + "totals": {"installs": 10.0, "network_installs": 0.0, "network_cost": 0.0, "network_ecpi": 0.0}, "warnings": [], } diff --git a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py index 081ab6d9c05b5..bd37653447352 100644 --- a/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-adjust/unit_tests/test_streams.py @@ -31,11 +31,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp requests_mock.get(url=report_url, status_code=200, json=mock_report_response) stream = get_stream_by_name("AdjustReport", config_pass) expected_parsed_record = { - "attr_dependency": { - "campaign_id_network": "unknown", - "partner_id": "-300", - "partner": "Organic" - }, + "attr_dependency": {"campaign_id_network": "unknown", "partner_id": "-300", "partner": "Organic"}, "app": "Test app", "partner_name": "Organic", "campaign": "unknown", @@ -44,7 +40,7 @@ def test_parse_response(requests_mock, config_pass, report_url, mock_report_resp "installs": "10", "network_installs": "0", "network_cost": "0.0", - "network_ecpi": "0.0" + "network_ecpi": "0.0", } records = [] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py index e9ee73055239b..3e0ae06c53b82 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_backoff_strategy.py @@ -10,13 +10,7 @@ from source_airtable.airtable_backoff_strategy import AirtableBackoffStrategy -@pytest.mark.parametrize( - "response_code, expected_backoff_time", - [ - (429, 30), - (404, None) - ] -) +@pytest.mark.parametrize("response_code, expected_backoff_time", [(429, 30), (404, None)]) def test_backoff_time(response_code, expected_backoff_time): mocked_logger = MagicMock(spec=logging.Logger) backoff = AirtableBackoffStrategy(logger=mocked_logger) diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py index 67daf8eb31296..613bd62185ae1 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_airtable_error_handler.py @@ -17,10 +17,18 @@ @pytest.mark.parametrize( "auth, json_response, error_message", [ - (TokenAuthenticator, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable"), - (AirtableOAuth, {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, "Access Token does not have required permissions, please reauthenticate."), - (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable.") - ] + ( + TokenAuthenticator, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable", + ), + ( + AirtableOAuth, + {"error": {"type": "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND"}}, + "Access Token does not have required permissions, please reauthenticate.", + ), + (TokenAuthenticator, {"error": {"type": "Test 403"}}, "Permission denied or entity is unprocessable."), + ], ) def test_interpret_response_handles_403_error(auth, json_response, error_message): mocked_authenticator = MagicMock(spec=auth) @@ -35,6 +43,7 @@ def test_interpret_response_handles_403_error(auth, json_response, error_message assert error_resolution.failure_type == FailureType.config_error assert error_resolution.error_message == error_message + def test_interpret_response_defers_to_airtable_error_mapping_for_other_errors(): mocked_logger = MagicMock(spec=logging.Logger) mocked_response = MagicMock(spec=Response) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py index 12579417383da..e1f8cd50883ff 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py @@ -25,10 +25,7 @@ def init_kwargs() -> Dict[str, Any]: @pytest.fixture def report_init_kwargs(init_kwargs) -> Dict[str, Any]: - return { - "stream_name": "GET_TEST_REPORT", - **init_kwargs - } + return {"stream_name": "GET_TEST_REPORT", **init_kwargs} @pytest.fixture diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py index 1225aa30b1812..6ca6ba55af131 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/test_report_based_streams.py @@ -447,14 +447,16 @@ def test_given_http_error_500_on_create_report_when_read_then_no_records_and_err @pytest.mark.parametrize(("stream_name", "data_format"), STREAMS) @HttpMocker() def test_given_http_error_not_support_account_id_of_type_vendor_when_read_then_no_records_and_error_logged( - self, stream_name: str, data_format: str, http_mocker: HttpMocker + self, stream_name: str, data_format: str, http_mocker: HttpMocker ): mock_auth(http_mocker) response_body = { "errors": [ - {"code": "InvalidInput", - "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", - "details": ""} + { + "code": "InvalidInput", + "message": "Report type 301 does not support account ID of type class com.amazon.partner.account.id.VendorGroupId.", + "details": "", + } ] } http_mocker.post( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py index 099f99f42f396..6eb648bc4a3b0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/integration/utils.py @@ -54,7 +54,11 @@ def get_stream_by_name(stream_name: str, config_: Mapping[str, Any]) -> Stream: def find_template(resource: str, execution_folder: str, template_format: Optional[str] = "csv") -> str: response_template_filepath = str( # FIXME: the below function should be replaced with the public version after next CDK release - _get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}.{template_format}" + _get_unit_test_folder(execution_folder) + / "resource" + / "http" + / "response" + / f"{resource}.{template_format}" ) with open(response_template_filepath, "r") as template_file: if template_file == "json": diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index 385699cc58440..36a1648a236c9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -211,15 +211,7 @@ def test_reports_read_records_raise_on_backoff(mocker, requests_mock, caplog): requests_mock.post( "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, ) stream = RestockInventoryReports( diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index a77cf2bd0b072..d0dc0c2df6e66 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -124,19 +124,23 @@ def test_check_connection_with_orders(requests_mock, connector_config_with_repor ( "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ - ("GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + ( + "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", [ {"option_name": "some_name_1", "option_value": "some_value_1"}, {"option_name": "some_name_2", "option_value": "some_value_2"}, ], - ), - ] + ), + ], ), ("SOME_OTHER_STREAM", []), ), ) def test_get_stream_report_options_list(connector_config_with_report_options, report_name, stream_name_w_options): - assert list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) == stream_name_w_options + assert ( + list(SourceAmazonSellerPartner().get_stream_report_kwargs(report_name, connector_config_with_report_options)) + == stream_name_w_options + ) def test_config_report_options_validation_error_duplicated_streams(connector_config_with_report_options): @@ -199,7 +203,9 @@ def test_spec(deployment_mode, common_streams_count, monkeypatch): "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", "GET_VENDOR_TRAFFIC_REPORT", } - streams_with_report_options = SourceAmazonSellerPartner().spec( - logger - ).connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + streams_with_report_options = ( + SourceAmazonSellerPartner() + .spec(logger) + .connectionSpecification["properties"]["report_options_list"]["items"]["properties"]["report_name"]["enum"] + ) assert len(set(streams_with_report_options).intersection(oss_only_streams)) == common_streams_count diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py index cae628254c611..54fa102d10363 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -246,15 +246,7 @@ def test_given_429_when_read_records_then_raise_transient_error(self, report_ini "POST", "https://test.url/reports/2021-06-30/reports", status_code=429, - json={ - "errors": [ - { - "code": "QuotaExceeded", - "message": "You exceeded your quota for the requested resource.", - "details": "" - } - ] - }, + json={"errors": [{"code": "QuotaExceeded", "message": "You exceeded your quota for the requested resource.", "details": ""}]}, reason="Forbidden", ) diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py index 7d78a625301a9..5fbe6f44f798c 100644 --- a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py @@ -141,9 +141,9 @@ def test_event_read(self, requests_mock): "error_code, expectation", [ (400, pytest.raises(AirbyteTracedException)), - (404, does_not_raise()), # does not raise because response action is IGNORE + (404, does_not_raise()), # does not raise because response action is IGNORE (504, pytest.raises(AirbyteTracedException)), - (500, does_not_raise()), # does not raise because repsonse action is RETRY + (500, does_not_raise()), # does not raise because repsonse action is RETRY ], ) def test_event_errors_read(self, mocker, requests_mock, error_code, expectation): diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py index 56efb7c45d89e..45d6acf5a69cb 100644 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-asana/unit_tests/test_config_migrations.py @@ -10,13 +10,17 @@ def test_should_migrate(): assert AsanaConfigMigration.should_migrate({"access_token": "asdfcxz"}) is True - assert AsanaConfigMigration.should_migrate({"credentials": { "option_title": "PAT Credentials", "personal_access_token": "1206938133417909" }} -) is False + assert ( + AsanaConfigMigration.should_migrate( + {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "1206938133417909"}} + ) + is False + ) def test__modify_and_save(): user_config = {"access_token": "asdfcxz"} - expected = {"credentials": { "option_title": "PAT Credentials", "personal_access_token": "asdfcxz" }} + expected = {"credentials": {"option_title": "PAT Credentials", "personal_access_token": "asdfcxz"}} # todo: need to make the migrate a classmethod instead of staticmethod since the missing config field will fail validation source = SourceAsana(config=user_config, catalog=None, state=None) diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py index 49f77bed58ec9..983984d337268 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py @@ -7,14 +7,14 @@ from source_avni.components import CustomAuthenticator -@patch('boto3.client') +@patch("boto3.client") def test_token_property(mock_boto3_client): mock_cognito_client = Mock() mock_boto3_client.return_value = mock_cognito_client - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") source._username = Mock() source._username.eval.return_value = "test_username" source._password = Mock() @@ -22,24 +22,19 @@ def test_token_property(mock_boto3_client): source.get_client_id = Mock() source.get_client_id.return_value = "test_client_id" - mock_cognito_client.initiate_auth.return_value = { - "AuthenticationResult": { - "IdToken": "test_id_token" - } - } + mock_cognito_client.initiate_auth.return_value = {"AuthenticationResult": {"IdToken": "test_id_token"}} token = source.token mock_boto3_client.assert_called_once_with("cognito-idp", region_name="ap-south-1") mock_cognito_client.initiate_auth.assert_called_once_with( - ClientId="test_client_id", - AuthFlow="USER_PASSWORD_AUTH", - AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} + ClientId="test_client_id", AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} ) assert token == "test_id_token" + def test_get_client_id(mocker): - - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") client_id = source.get_client_id() expected_length = 26 - assert len(client_id) == expected_length \ No newline at end of file + assert len(client_id) == expected_length diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py index d66169a9d7ae6..715a7466d4186 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config_migrations.py @@ -15,13 +15,11 @@ class MigrateConfig(ABC): @classmethod @abstractmethod - def should_migrate(cls, config: Mapping[str, Any]) -> bool: - ... + def should_migrate(cls, config: Mapping[str, Any]) -> bool: ... @classmethod @abstractmethod - def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: - ... + def migrate_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: ... @classmethod def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py index dbe30bfeb7fe8..b81b0b4b855fe 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_authenticator.py @@ -8,18 +8,18 @@ def test_custom_authenticator(requests_mock): authenticator = AzureOauth2Authenticator( - token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", - client_id="client_id", - client_secret="client_secret", - refresh_token="refresh_token", - ) + token_refresh_endpoint="https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + client_id="client_id", + client_secret="client_secret", + refresh_token="refresh_token", + ) token_refresh_response = { "token_type": "Bearer", "scope": "https://storage.azure.com/user_impersonation https://storage.azure.com/.default", "expires_in": 5144, "ext_expires_in": 5144, "access_token": "access_token", - "refresh_token": "refresh_token" + "refresh_token": "refresh_token", } requests_mock.post("https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", json=token_refresh_response) new_token = authenticator.get_token() diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py index 3cec8f051aa2f..86313782cc1ab 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/test_stream_reader.py @@ -18,18 +18,19 @@ @pytest.mark.parametrize( "credentials, expected_credentials_type", [ - ({"auth_type": "oauth2", - "tenant_id": "tenant_id", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token" - }, AzureOauth2Authenticator), - ({ - "auth_type": "storage_account_key", - "azure_blob_storage_account_key": "key1" - }, str), + ( + { + "auth_type": "oauth2", + "tenant_id": "tenant_id", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, + AzureOauth2Authenticator, + ), + ({"auth_type": "storage_account_key", "azure_blob_storage_account_key": "key1"}, str), ], - ids=["oauth2", "storage_account_key"] + ids=["oauth2", "storage_account_key"], ) def test_stream_reader_credentials(credentials: Dict, expected_credentials_type: Union[str, AzureOauth2Authenticator]): reader = SourceAzureBlobStorageStreamReader() @@ -59,9 +60,9 @@ def test_stream_reader_files_read_and_filter_by_date(): reader.config = config with patch.object(ContainerClient, "list_blobs") as blobs: blobs.return_value = [ - BlobProperties(name='sample_file_1.csv', **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_2.csv', **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), - BlobProperties(name='sample_file_3.csv', **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}) + BlobProperties(name="sample_file_1.csv", **{"Last-Modified": datetime.datetime(2023, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_2.csv", **{"Last-Modified": datetime.datetime(2024, 1, 1, 1, 1, 0)}), + BlobProperties(name="sample_file_3.csv", **{"Last-Modified": datetime.datetime(2024, 1, 5, 1, 1, 0)}), ] files = list(reader.get_matching_files(globs=["**"], prefix=None, logger=logger)) assert len(files) == 2 diff --git a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py index 752e8472480cf..aa1b2aaceb788 100644 --- a/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py +++ b/airbyte-integrations/connectors/source-azure-table/unit_tests/test_azure_table.py @@ -23,14 +23,11 @@ def test_get_table_service_client_handles_exception(mocker, reader): """ Test that get_table_service_client method handles exceptions correctly. """ - mocker.patch( - "source_azure_table.azure_table.TableServiceClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableServiceClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_service_client() - + assert "Connection error" in str(exc_info.value) @@ -58,10 +55,7 @@ def test_get_table_client_handles_exception(mocker, reader): reader.get_table_client("") assert "table name is not valid." in str(exc_info.value) - mocker.patch( - "source_azure_table.azure_table.TableClient.from_connection_string", - side_effect=Exception("Connection error") - ) + mocker.patch("source_azure_table.azure_table.TableClient.from_connection_string", side_effect=Exception("Connection error")) with pytest.raises(Exception) as exc_info: reader.get_table_client("valid_table_name") @@ -74,10 +68,7 @@ def test_get_tables_return(mocker, reader, tables): """ mock_client = mocker.MagicMock() mock_client.list_tables.return_value = tables.__iter__() - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) result = reader.get_tables() result_table_names = [table.name for table in result] @@ -92,10 +83,7 @@ def test_get_tables_handles_exception(mocker, reader): """ mock_client = mocker.MagicMock() mock_client.list_tables.side_effect = Exception("Failed to list tables") - mocker.patch( - "azure.data.tables.TableServiceClient.from_connection_string", - return_value=mock_client - ) + mocker.patch("azure.data.tables.TableServiceClient.from_connection_string", return_value=mock_client) with pytest.raises(Exception) as exc_info: reader.get_tables() diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py index 1dd16c11461f2..b6790c8b6e397 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ad_labels_stream.py @@ -38,7 +38,9 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-04T12:12:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-04T12:12:12.028+00:00" + } @HttpMocker() @freeze_time("2024-02-26") # mock current time as stream data available for 30 days only @@ -46,14 +48,14 @@ def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ad_labels_state", self.stream_name) self.auth_client(http_mocker) output, service_call_mock = self.read_stream( - self.stream_name, - SyncMode.incremental, - self._config, - "app_install_ad_labels_with_state", - state + self.stream_name, SyncMode.incremental, self._config, "app_install_ad_labels_with_state", state ) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-29T12:55:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-29T12:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py index 7524b3241110c..5d91710695cd8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/integrations/test_app_install_ads_stream.py @@ -38,16 +38,24 @@ def test_incremental_read_cursor_value_matches_value_from_most_recent_record(sel self.auth_client(http_mocker) output, _ = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_cursor_value") assert len(output.records) == 4 - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-03-01T12:49:12.028+00:00"} + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-03-01T12:49:12.028+00:00" + } @HttpMocker() @freeze_time("2023-12-29") # mock current time as stream data available for 30 days only def test_incremental_read_with_state(self, http_mocker: HttpMocker): state = self._state("app_install_ads_state", self.stream_name) self.auth_client(http_mocker) - output, service_call_mock = self.read_stream(self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state) - assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == {self.cursor_field: "2024-01-01T10:55:12.028+00:00"} + output, service_call_mock = self.read_stream( + self.stream_name, SyncMode.incremental, self._config, "app_install_ads_with_state", state + ) + assert output.most_recent_state.stream_state.__dict__.get(self.account_id, {}) == { + self.cursor_field: "2024-01-01T10:55:12.028+00:00" + } previous_state = state[0].stream.stream_state.__dict__ # gets DownloadParams object - assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse(previous_state[self.account_id][self.cursor_field]) + assert service_call_mock.call_args.args[0].last_sync_time_in_utc == pendulum.parse( + previous_state[self.account_id][self.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py index 9a96becee8c61..7fc12f901ef17 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -88,18 +88,18 @@ def test_bulk_stream_read_with_chunks_app_install_ad_labels(mocked_client, confi app_install_ads = AppInstallAdLabels(mocked_client, config) result = app_install_ads.read_with_chunks(path=path_to_file) assert next(result) == { - 'Ad Group': None, - 'Campaign': None, - 'Client Id': 'ClientIdGoesHere', - 'Color': None, - 'Description': None, - 'Id': '-22', - 'Label': None, - 'Modified Time': None, - 'Name': None, - 'Parent Id': '-11112', - 'Status': None, - 'Type': 'App Install Ad Label' + "Ad Group": None, + "Campaign": None, + "Client Id": "ClientIdGoesHere", + "Color": None, + "Description": None, + "Id": "-22", + "Label": None, + "Modified Time": None, + "Name": None, + "Parent Id": "-11112", + "Status": None, + "Type": "App Install Ad Label", } @@ -116,17 +116,21 @@ def test_bulk_stream_read_with_chunks_ioe_error(mocked_client, config, caplog): @pytest.mark.parametrize( "stream_state, config_start_date, expected_start_date", [ - ({"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC)), + ( + {"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, + "2020-01-01", + DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC), + ), ({"another_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", None), ({}, "2020-01-01", None), ({}, "2023-10-21", DateTime(2023, 10, 21, 0, 0, 0, tzinfo=UTC)), ], - ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"] + ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"], ) def test_bulk_stream_start_date(mocked_client, config, stream_state, config_start_date, expected_start_date): mocked_client.reports_start_date = pendulum.parse(config_start_date) if config_start_date else None stream = AppInstallAds(mocked_client, config) - assert expected_start_date == stream.get_start_date(stream_state, 'some_account_id') + assert expected_start_date == stream.get_start_date(stream_state, "some_account_id") @patch.object(source_bing_ads.source, "Client") @@ -140,18 +144,10 @@ def test_bulk_stream_stream_state(mocked_client, config): assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} # stream state saved to connection state stream.state = { - "120342748234": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "27364572345": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "732645723": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - }, - "837563864": { - "Modified Time": "2022-11-05T12:07:29.360+00:00" - } + "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "27364572345": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "732645723": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "837563864": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, } assert stream.state == { "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, @@ -165,4 +161,6 @@ def test_bulk_stream_stream_state(mocked_client, config): @patch.object(source_bing_ads.source, "Client") def test_bulk_stream_custom_transform_date_rfc3339(mocked_client, config): stream = AppInstallAds(mocked_client, config) - assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339("04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field]) + assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339( + "04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field] + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index f68acb43a302d..737fc7e77277c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -259,7 +259,10 @@ def test_report_parse_response_csv_error(caplog): fake_response = MagicMock() fake_response.report_records.__iter__ = MagicMock(side_effect=_csv.Error) list(stream_report.parse_response(fake_response)) - assert "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." in caplog.messages + assert ( + "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." + in caplog.messages + ) @patch.object(source_bing_ads.source, "Client") @@ -404,10 +407,10 @@ def test_account_performance_report_monthly_stream_slices(mocked_client, config_ with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, - {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'LastYear'}, - {'account_id': 180278106, 'customer_id': 200, 'time_period': 'ThisYear'} + {"account_id": 180519267, "customer_id": 100, "time_period": "LastYear"}, + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "LastYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, ] @@ -417,10 +420,7 @@ def test_account_performance_report_monthly_stream_slices_no_time_period(mocked_ accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) with patch.object(Accounts, "read_records", return_value=accounts_read_records): stream_slice = list(account_performance_report_monthly.stream_slices(sync_mode=SyncMode.full_refresh)) - assert stream_slice == [ - {'account_id': 180519267, 'customer_id': 100}, - {'account_id': 180278106, 'customer_id': 200} - ] + assert stream_slice == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] @pytest.mark.parametrize( @@ -451,14 +451,38 @@ def test_custom_performance_report_no_last_year_stream_slices(mocked_client, con (AccountPerformanceReportHourly, "hourly_reports/account_performance.csv", "hourly_reports/account_performance_records.json"), (AdGroupPerformanceReportHourly, "hourly_reports/ad_group_performance.csv", "hourly_reports/ad_group_performance_records.json"), (AdPerformanceReportHourly, "hourly_reports/ad_performance.csv", "hourly_reports/ad_performance_records.json"), - (CampaignImpressionPerformanceReportHourly, "hourly_reports/campaign_impression_performance.csv", "hourly_reports/campaign_impression_performance_records.json"), + ( + CampaignImpressionPerformanceReportHourly, + "hourly_reports/campaign_impression_performance.csv", + "hourly_reports/campaign_impression_performance_records.json", + ), (KeywordPerformanceReportHourly, "hourly_reports/keyword_performance.csv", "hourly_reports/keyword_performance_records.json"), - (GeographicPerformanceReportHourly, "hourly_reports/geographic_performance.csv", "hourly_reports/geographic_performance_records.json"), + ( + GeographicPerformanceReportHourly, + "hourly_reports/geographic_performance.csv", + "hourly_reports/geographic_performance_records.json", + ), (AgeGenderAudienceReportHourly, "hourly_reports/age_gender_audience.csv", "hourly_reports/age_gender_audience_records.json"), - (SearchQueryPerformanceReportHourly, "hourly_reports/search_query_performance.csv", "hourly_reports/search_query_performance_records.json"), - (UserLocationPerformanceReportHourly, "hourly_reports/user_location_performance.csv", "hourly_reports/user_location_performance_records.json"), - (AccountImpressionPerformanceReportHourly, "hourly_reports/account_impression_performance.csv", "hourly_reports/account_impression_performance_records.json"), - (AdGroupImpressionPerformanceReportHourly, "hourly_reports/ad_group_impression_performance.csv", "hourly_reports/ad_group_impression_performance_records.json"), + ( + SearchQueryPerformanceReportHourly, + "hourly_reports/search_query_performance.csv", + "hourly_reports/search_query_performance_records.json", + ), + ( + UserLocationPerformanceReportHourly, + "hourly_reports/user_location_performance.csv", + "hourly_reports/user_location_performance_records.json", + ), + ( + AccountImpressionPerformanceReportHourly, + "hourly_reports/account_impression_performance.csv", + "hourly_reports/account_impression_performance_records.json", + ), + ( + AdGroupImpressionPerformanceReportHourly, + "hourly_reports/ad_group_impression_performance.csv", + "hourly_reports/ad_group_impression_performance_records.json", + ), ], ) @patch.object(source_bing_ads.source, "Client") diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py index 85f0de928865a..65fdee1495555 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/config.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "site": "ConfigBuilder default site", "site_api_key": "ConfigBuilder default site api key", "start_date": "2023-01-01T06:57:44Z", - "product_catalog": "2.0" + "product_catalog": "2.0", } def with_site(self, site: str) -> "ConfigBuilder": @@ -30,4 +30,4 @@ def with_product_catalog(self, product_catalog: str) -> "ConfigBuilder": return self def build(self) -> Dict[str, Any]: - return self._config \ No newline at end of file + return self._config diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py index 0cf9d9d5a5bcd..f5cd872ef7c05 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/pagination.py @@ -8,4 +8,4 @@ class ChargebeePaginationStrategy(PaginationStrategy): @staticmethod def update(response: Dict[str, Any]) -> None: - response["next_offset"] = "[1707076198000,57873868]" \ No newline at end of file + response["next_offset"] = "[1707076198000,57873868]" diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py index 1b97d9e4d6fb1..8e30510214be3 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/request_builder.py @@ -69,7 +69,7 @@ def with_include_deleted(self, include_deleted: bool) -> "ChargebeeRequestBuilde return self def with_created_at_btw(self, created_at_btw: List[int]) -> "ChargebeeRequestBuilder": - self._created_at_btw = f'{created_at_btw}' + self._created_at_btw = f"{created_at_btw}" return self def with_updated_at_btw(self, updated_at_btw: List[int]) -> "ChargebeeRequestBuilder": @@ -97,7 +97,7 @@ def with_limit(self, limit: int) -> "ChargebeeRequestBuilder": return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -117,7 +117,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( @@ -126,6 +128,7 @@ def build(self) -> HttpRequest: headers={"Authorization": f"Basic {base64.b64encode((str(self._site_api_key) + ':').encode('utf-8')).decode('utf-8')}"}, ) + class ChargebeeSubstreamRequestBuilder(ChargebeeRequestBuilder): @classmethod @@ -141,7 +144,7 @@ def with_endpoint_path(self, endpoint_path: str) -> "ChargebeeSubstreamRequestBu return self def build(self) -> HttpRequest: - query_params= {} + query_params = {} if self._sort_by_asc: query_params["sort_by[asc]"] = self._sort_by_asc if self._sort_by_desc: @@ -161,7 +164,9 @@ def build(self) -> HttpRequest: if self._any_query_params: if query_params: - raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + raise ValueError( + f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both." + ) query_params = ANY_QUERY_PARAMS return HttpRequest( diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py index f9163b6be3a87..838fb8d82dc78 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/response_builder.py @@ -10,5 +10,6 @@ def a_response_with_status(status_code: int) -> HttpResponse: return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code) + def a_response_with_status_and_header(status_code: int, header: Mapping[str, str]) -> HttpResponse: - return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) \ No newline at end of file + return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code, header) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py index 91ee4bd28dec1..75ef457f0f164 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_addon.py @@ -57,28 +57,25 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() source = _source(catalog=catalog, config=config, state=state) return read(source, config, catalog, state, expecting_exception) + @freezegun.freeze_time(_NOW.isoformat()) class FullRefreshTest(TestCase): @@ -96,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -106,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -120,14 +125,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -171,6 +172,7 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): @@ -189,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -202,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py index 0c0d2288b502e..38ecd78e73083 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_coupon.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +103,14 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +121,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +130,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -202,8 +196,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,7 +208,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py index 18c98133aea86..11be0264aa3d8 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_customer.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +127,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +136,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -183,6 +183,7 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): @@ -201,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -214,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py index 92323100d784b..c365b520b2965 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_event.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -109,11 +103,15 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht # Tests pagination http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_occurred_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -187,8 +185,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -200,7 +197,7 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_sort_by_asc(_CURSOR_FIELD).with_occurred_at_btw([state_cursor_value, self._now_in_seconds]).build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py index e0fe9d45d38a8..394bed3d5434e 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_hosted_page.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -168,6 +171,7 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): @@ -186,8 +190,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -199,9 +202,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py index ed00c2677e244..8357c114fe368 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_plan.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,10 +125,7 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py index cf8e8b67164d2..bb518a7871d88 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_site_migration_detail.py @@ -35,9 +35,9 @@ _NO_STATE = {} _NOW = datetime.now(timezone.utc) -''' +""" Note that this is a semi-incremental stream and tests will need to be adapated accordingly -''' +""" def _a_request() -> ChargebeeRequestBuilder: @@ -61,22 +61,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) + def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -101,8 +97,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -110,26 +105,16 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock @HttpMocker() def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination - http_mocker.get( - _a_request().build(), - _a_response().with_record(_a_record()).with_pagination().build() - ) - http_mocker.get( - _a_request().with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() - ) + http_mocker.get(_a_request().build(), _a_response().with_record(_a_record()).with_pagination().build()) + http_mocker.get(_a_request().with_offset("[1707076198000,57873868]").build(), _a_response().with_record(_a_record()).build()) self._read(_config().with_start_date(self._start_date)) # HTTPMocker ensures call are performed - @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -197,8 +182,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date), _NO_STATE) most_recent_state = output.most_recent_state @@ -209,7 +193,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_fiel def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recent_cursor_value(self, http_mocker: HttpMocker) -> None: state_cursor_value = self._start_date_in_seconds + 1 record_cursor_value = state_cursor_value + 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -219,13 +203,17 @@ def test_given_initial_state_value_when_read_then_state_is_updated_to_most_recen output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=record_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) @HttpMocker() - def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated(self, http_mocker: HttpMocker) -> None: + def test_given_record_returned_with_cursor_value_before_state_record_is_not_read_and_state_not_updated( + self, http_mocker: HttpMocker + ) -> None: state_cursor_value = self._start_date_in_seconds record_cursor_value = self._start_date_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( _a_request().with_any_query_params().build(), @@ -235,5 +223,7 @@ def test_given_record_returned_with_cursor_value_before_state_record_is_not_read output = self._read(_config().with_start_date(self._start_date), state) most_recent_state = output.most_recent_state assert most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) - assert most_recent_state.stream_state == AirbyteStateBlob(migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value}) + assert most_recent_state.stream_state == AirbyteStateBlob( + migrated_at=state_cursor_value, prior_state={_CURSOR_FIELD: state_cursor_value} + ) assert len(output.records) == 0 diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py index 7b980fa4fbf8a..888311470ecdf 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date)) assert len(output.records) == 2 @@ -108,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -124,7 +127,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock # Tests custom field transformation http_mocker.get( _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build() + _a_response().with_record(_a_record().with_field(NestedPath([_STREAM_NAME, "cf_my_custom_field"]), "my_custom_value")).build(), ) output = self._read(_config().with_start_date(self._start_date)) assert output.records[0].record.data["custom_fields"][0]["name"] == "cf_my_custom_field" @@ -133,10 +136,7 @@ def test_given_records_returned_with_custom_field_transformation(self, http_mock @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 @@ -202,8 +202,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -215,9 +214,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py index f0ad3b8c62255..603630cbf5c9a 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_subscription_with_scheduled_changes.py @@ -62,7 +62,7 @@ def _a_parent_record() -> RecordBuilder: find_template("subscription", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) @@ -71,31 +71,24 @@ def _a_child_record() -> RecordBuilder: find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), record_id_path=NestedPath(["subscription", _PRIMARY_KEY]), - record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["subscription", _CURSOR_FIELD]), ) def _a_parent_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _a_child_response() -> HttpResponseBuilder: return create_response_builder( - find_template("subscription_with_scheduled_changes", __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template("subscription_with_scheduled_changes", __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -122,11 +115,15 @@ def test_when_read_then_records_are_extracted(self, http_mocker: HttpMocker) -> http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -137,19 +134,29 @@ def test_given_multiple_parents_when_read_then_fetch_for_each_parent(self, http_ a_parent_id = "a_subscription_test" another_parent_id = "another_subscription_test" - http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(a_parent_id)).with_record(_a_parent_record().with_id(another_parent_id)).build() + _a_parent_response() + .with_record(_a_parent_record().with_id(a_parent_id)) + .with_record(_a_parent_record().with_id(another_parent_id)) + .build(), ) http_mocker.get( - _a_child_request().with_parent_id(a_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(a_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) http_mocker.get( - _a_child_request().with_parent_id(another_parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(another_parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -161,11 +168,15 @@ def test_when_read_then_primary_key_is_set(self, http_mocker: HttpMocker) -> Non http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), - _a_child_response().with_record(_a_child_record()).build() + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), + _a_child_response().with_record(_a_child_record()).build(), ) output = self._read(_config().with_start_date(self._start_date)) @@ -178,10 +189,14 @@ def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocke http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(400), ) @@ -194,10 +209,14 @@ def test_given_http_status_404_when_read_then_stream_is_ignored(self, http_mocke http_mocker.get( _a_parent_request().with_any_query_params().build(), - _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build() + _a_parent_response().with_record(_a_parent_record().with_id(parent_id)).build(), ) http_mocker.get( - _a_child_request().with_parent_id(parent_id).with_endpoint_path("retrieve_with_scheduled_changes").with_any_query_params().build(), + _a_child_request() + .with_parent_id(parent_id) + .with_endpoint_path("retrieve_with_scheduled_changes") + .with_any_query_params() + .build(), a_response_with_status(404), ) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py index 8ee8f4a9d3413..c9d2409d8d65b 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/integration/test_virtual_bank_account.py @@ -57,23 +57,18 @@ def _a_record() -> RecordBuilder: find_template(_STREAM_NAME, __file__), FieldPath("list"), record_id_path=NestedPath([_STREAM_NAME, _PRIMARY_KEY]), - record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]) + record_cursor_path=NestedPath([_STREAM_NAME, _CURSOR_FIELD]), ) def _a_response() -> HttpResponseBuilder: return create_response_builder( - find_template(_STREAM_NAME, __file__), - FieldPath("list"), - pagination_strategy=ChargebeePaginationStrategy() + find_template(_STREAM_NAME, __file__), FieldPath("list"), pagination_strategy=ChargebeePaginationStrategy() ) def _read( - config_builder: ConfigBuilder, - sync_mode: SyncMode, - state: Optional[Dict[str, Any]] = None, - expecting_exception: bool = False + config_builder: ConfigBuilder, sync_mode: SyncMode, state: Optional[Dict[str, Any]] = None, expecting_exception: bool = False ) -> EntrypointOutput: catalog = _catalog(sync_mode) config = config_builder.build() @@ -98,8 +93,7 @@ def _read(config: ConfigBuilder, expecting_exception: bool = False) -> Entrypoin def test_given_valid_response_records_are_extracted_and_returned(self, http_mocker: HttpMocker) -> None: # Tests simple read and record extraction http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record()).with_record(_a_record()).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record()).with_record(_a_record()).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8))) assert len(output.records) == 2 @@ -108,12 +102,21 @@ def test_given_valid_response_records_are_extracted_and_returned(self, http_mock def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: HttpMocker) -> None: # Tests pagination http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).build(), - _a_response().with_record(_a_record()).with_pagination().build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .build(), + _a_response().with_record(_a_record()).with_pagination().build(), ) http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]).with_offset("[1707076198000,57873868]").build(), - _a_response().with_record(_a_record()).build() + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([self._start_date_in_seconds, self._now_in_seconds]) + .with_offset("[1707076198000,57873868]") + .build(), + _a_response().with_record(_a_record()).build(), ) self._read(_config().with_start_date(self._start_date)) @@ -122,14 +125,10 @@ def test_given_multiple_pages_of_records_read_and_returned(self, http_mocker: Ht @HttpMocker() def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: # Tests 400 status error handling - http_mocker.get( - _a_request().with_any_query_params().build(), - a_response_with_status(400) - ) + http_mocker.get(_a_request().with_any_query_params().build(), a_response_with_status(400)) output = self._read(_config().with_start_date(self._start_date), expecting_exception=True) assert len(output.get_stream_statuses(f"{_STREAM_NAME}s")) == 0 - @HttpMocker() def test_given_http_status_401_when_the_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: # Test 401 status error handling @@ -173,6 +172,7 @@ def test_given_http_status_500_after_max_retries_raises_config_error(self, http_ output = self._read(_config(), expecting_exception=True) assert output.errors[-1].trace.error.failure_type == FailureType.config_error + @freezegun.freeze_time(_NOW.isoformat()) class IncrementalTest(TestCase): @@ -191,8 +191,7 @@ def test_given_no_initial_state_when_read_then_return_state_based_on_most_recent # Tests setting state when no initial state is provided cursor_value = self._start_date_in_seconds + 1 http_mocker.get( - _a_request().with_any_query_params().build(), - _a_response().with_record(_a_record().with_cursor(cursor_value)).build() + _a_request().with_any_query_params().build(), _a_response().with_record(_a_record().with_cursor(cursor_value)).build() ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), _NO_STATE) most_recent_state = output.most_recent_state @@ -204,9 +203,13 @@ def test_given_initial_state_use_state_for_query_params(self, http_mocker: HttpM # Tests updating query param with state state_cursor_value = int((self._now - timedelta(days=5)).timestamp()) record_cursor_value = self._now_in_seconds - 1 - state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() + state = StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: state_cursor_value}).build() http_mocker.get( - _a_request().with_sort_by_asc(_CURSOR_FIELD).with_include_deleted(True).with_updated_at_btw([state_cursor_value, self._now_in_seconds]).build(), + _a_request() + .with_sort_by_asc(_CURSOR_FIELD) + .with_include_deleted(True) + .with_updated_at_btw([state_cursor_value, self._now_in_seconds]) + .build(), _a_response().with_record(_a_record().with_cursor(record_cursor_value)).build(), ) output = self._read(_config().with_start_date(self._start_date - timedelta(hours=8)), state) diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py index 54709f56fed03..bdf4091dcbc9d 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py @@ -26,11 +26,12 @@ def test_field_transformation(record, expected_record): transformed_record = transformer.transform(record) assert transformed_record == expected_record + @pytest.mark.parametrize( "record_data, expected", [ ({"pk": 1, "name": "example", "updated_at": 1662459011}, True), - ] + ], ) def test_slicer(record_data, expected): date_time_dict = {"updated_at": 1662459010} @@ -50,24 +51,17 @@ def test_slicer(record_data, expected): @pytest.mark.parametrize( "first_record, second_record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662460000}, - True), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2", "updated_at": 1662440000}, - False), - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - {"pk": 2, "name": "example2"}, - False), - ({"pk": 1, "name": "example"}, - {"pk": 2, "name": "example2", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662460000}, True), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2", "updated_at": 1662440000}, False), + ({"pk": 1, "name": "example", "updated_at": 1662459010}, {"pk": 2, "name": "example2"}, False), + ({"pk": 1, "name": "example"}, {"pk": 2, "name": "example2", "updated_at": 1662459010}, True), + ], ) def test_is_greater_than_or_equal(first_record, second_record, expected): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") assert slicer.is_greater_than_or_equal(second_record, first_record) == expected + def test_set_initial_state(): cursor_field = "updated_at" cursor_value = 999999999 @@ -75,18 +69,19 @@ def test_set_initial_state(): slicer.set_initial_state(stream_state={cursor_field: cursor_value}) assert slicer._state[cursor_field] == cursor_value + @pytest.mark.parametrize( "record, expected", [ - ({"pk": 1, "name": "example", "updated_at": 1662459010}, - True), - ] + ({"pk": 1, "name": "example", "updated_at": 1662459010}, True), + ], ) def test_should_be_synced(record, expected): cursor_field = "updated_at" slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field=cursor_field) assert slicer.should_be_synced(record) == expected + def test_stream_slices(): slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") stream_slices_instance = slicer.stream_slices() diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py index a6f3a7bc3a6f5..3d810f28a754c 100644 --- a/airbyte-integrations/connectors/source-commcare/source_commcare/source.py +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/source.py @@ -136,7 +136,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Case(IncrementalStream): - """ docs: https://www.commcarehq.org/a/[domain]/api/[version]/case/ """ diff --git a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py index e2f03e369f8db..4c54c7f425383 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py +++ b/airbyte-integrations/connectors/source-declarative-manifest/unit_tests/test_source_declarative_local_manifest.py @@ -16,8 +16,10 @@ @pytest.fixture(autouse=True) def setup(valid_local_manifest_yaml): - with patch('source_declarative_manifest.run._is_local_manifest_command', return_value=True): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=valid_local_manifest_yaml): + with patch("source_declarative_manifest.run._is_local_manifest_command", return_value=True): + with patch( + "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=valid_local_manifest_yaml + ): yield @@ -28,9 +30,9 @@ def test_spec_is_poke_api(capsys): def test_invalid_yaml_throws(capsys, invalid_local_manifest_yaml): - with patch('source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file', return_value=invalid_local_manifest_yaml): - with pytest.raises(ValidationError): - run.handle_command(["spec"]) + with patch("source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=invalid_local_manifest_yaml): + with pytest.raises(ValidationError): + run.handle_command(["spec"]) def test_given_invalid_config_then_unsuccessful_check(capsys, invalid_local_config_file): diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index b33b9278eeb72..f9738fc5cbe16 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -214,9 +214,11 @@ def state(self, value: Mapping[str, Any]): return self._cursor_values = { - account_id: pendulum.parse(transformed_state[account_id][self.cursor_field]).date() - if transformed_state.get(account_id, {}).get(self.cursor_field) - else None + account_id: ( + pendulum.parse(transformed_state[account_id][self.cursor_field]).date() + if transformed_state.get(account_id, {}).get(self.cursor_field) + else None + ) for account_id in self._account_ids } self._completed_slices = { diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index c3446cdfea01f..eef1d13e164b9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -91,7 +91,8 @@ def give_up(details): def is_transient_cannot_include_error(exc: FacebookRequestError) -> bool: """After migration to API v19.0, some customers randomly face a BAD_REQUEST error (OAuthException) with the pattern:"Cannot include ..." - According to the last comment in https://developers.facebook.com/community/threads/286697364476462/, this might be a transient issue that can be solved with a retry.""" + According to the last comment in https://developers.facebook.com/community/threads/286697364476462/, this might be a transient issue that can be solved with a retry. + """ pattern = r"Cannot include .* in summary param because they weren't there while creating the report run." return bool(exc.http_status() == http.client.BAD_REQUEST and re.search(pattern, exc.api_error_message())) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 2a290cd48cbc6..f4df9b56e420a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -36,108 +36,108 @@ _REPORT_RUN_ID = "1571860060019548" _JOB_ID = "1049937379601625" _JOB_START_FIELDS = [ - "account_currency", - "account_id", - "account_name", - "action_values", - "actions", - "ad_click_actions", - "ad_id", - "ad_impression_actions", - "ad_name", - "adset_id", - "adset_name", - "attribution_setting", - "auction_bid", - "auction_competitiveness", - "auction_max_competitor_bid", - "buying_type", - "campaign_id", - "campaign_name", - "canvas_avg_view_percent", - "canvas_avg_view_time", - "catalog_segment_actions", - "catalog_segment_value", - "catalog_segment_value_mobile_purchase_roas", - "catalog_segment_value_omni_purchase_roas", - "catalog_segment_value_website_purchase_roas", - "clicks", - "conversion_rate_ranking", - "conversion_values", - "conversions", - "converted_product_quantity", - "converted_product_value", - "cost_per_15_sec_video_view", - "cost_per_2_sec_continuous_video_view", - "cost_per_action_type", - "cost_per_ad_click", - "cost_per_conversion", - "cost_per_estimated_ad_recallers", - "cost_per_inline_link_click", - "cost_per_inline_post_engagement", - "cost_per_outbound_click", - "cost_per_thruplay", - "cost_per_unique_action_type", - "cost_per_unique_click", - "cost_per_unique_inline_link_click", - "cost_per_unique_outbound_click", - "cpc", - "cpm", - "cpp", - "created_time", - "ctr", - "date_start", - "date_stop", - "engagement_rate_ranking", - "estimated_ad_recallers", - "frequency", - "full_view_impressions", - "full_view_reach", - "impressions", - "inline_link_click_ctr", - "inline_link_clicks", - "inline_post_engagement", - "instant_experience_clicks_to_open", - "instant_experience_clicks_to_start", - "instant_experience_outbound_clicks", - "mobile_app_purchase_roas", - "objective", - "optimization_goal", - "outbound_clicks", - "outbound_clicks_ctr", - "purchase_roas", - "qualifying_question_qualify_answer_rate", - "quality_ranking", - "reach", - "social_spend", - "spend", - "unique_actions", - "unique_clicks", - "unique_ctr", - "unique_inline_link_click_ctr", - "unique_inline_link_clicks", - "unique_link_clicks_ctr", - "unique_outbound_clicks", - "unique_outbound_clicks_ctr", - "updated_time", - "video_15_sec_watched_actions", - "video_30_sec_watched_actions", - "video_avg_time_watched_actions", - "video_continuous_2_sec_watched_actions", - "video_p100_watched_actions", - "video_p25_watched_actions", - "video_p50_watched_actions", - "video_p75_watched_actions", - "video_p95_watched_actions", - "video_play_actions", - "video_play_curve_actions", - "video_play_retention_0_to_15s_actions", - "video_play_retention_20_to_60s_actions", - "video_play_retention_graph_actions", - "video_time_watched_actions", - "website_ctr", - "website_purchase_roas", - ] + "account_currency", + "account_id", + "account_name", + "action_values", + "actions", + "ad_click_actions", + "ad_id", + "ad_impression_actions", + "ad_name", + "adset_id", + "adset_name", + "attribution_setting", + "auction_bid", + "auction_competitiveness", + "auction_max_competitor_bid", + "buying_type", + "campaign_id", + "campaign_name", + "canvas_avg_view_percent", + "canvas_avg_view_time", + "catalog_segment_actions", + "catalog_segment_value", + "catalog_segment_value_mobile_purchase_roas", + "catalog_segment_value_omni_purchase_roas", + "catalog_segment_value_website_purchase_roas", + "clicks", + "conversion_rate_ranking", + "conversion_values", + "conversions", + "converted_product_quantity", + "converted_product_value", + "cost_per_15_sec_video_view", + "cost_per_2_sec_continuous_video_view", + "cost_per_action_type", + "cost_per_ad_click", + "cost_per_conversion", + "cost_per_estimated_ad_recallers", + "cost_per_inline_link_click", + "cost_per_inline_post_engagement", + "cost_per_outbound_click", + "cost_per_thruplay", + "cost_per_unique_action_type", + "cost_per_unique_click", + "cost_per_unique_inline_link_click", + "cost_per_unique_outbound_click", + "cpc", + "cpm", + "cpp", + "created_time", + "ctr", + "date_start", + "date_stop", + "engagement_rate_ranking", + "estimated_ad_recallers", + "frequency", + "full_view_impressions", + "full_view_reach", + "impressions", + "inline_link_click_ctr", + "inline_link_clicks", + "inline_post_engagement", + "instant_experience_clicks_to_open", + "instant_experience_clicks_to_start", + "instant_experience_outbound_clicks", + "mobile_app_purchase_roas", + "objective", + "optimization_goal", + "outbound_clicks", + "outbound_clicks_ctr", + "purchase_roas", + "qualifying_question_qualify_answer_rate", + "quality_ranking", + "reach", + "social_spend", + "spend", + "unique_actions", + "unique_clicks", + "unique_ctr", + "unique_inline_link_click_ctr", + "unique_inline_link_clicks", + "unique_link_clicks_ctr", + "unique_outbound_clicks", + "unique_outbound_clicks_ctr", + "updated_time", + "video_15_sec_watched_actions", + "video_30_sec_watched_actions", + "video_avg_time_watched_actions", + "video_continuous_2_sec_watched_actions", + "video_p100_watched_actions", + "video_p25_watched_actions", + "video_p50_watched_actions", + "video_p75_watched_actions", + "video_p95_watched_actions", + "video_play_actions", + "video_play_curve_actions", + "video_play_retention_0_to_15s_actions", + "video_play_retention_20_to_60s_actions", + "video_play_retention_graph_actions", + "video_time_watched_actions", + "website_ctr", + "website_purchase_roas", +] def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) -> RequestBuilder: @@ -145,7 +145,10 @@ def _update_api_throttle_limit_request(account_id: Optional[str] = ACCOUNT_ID) - def _job_start_request( - account_id: Optional[str] = ACCOUNT_ID, since: Optional[datetime] = None, until: Optional[datetime] = None, fields: Optional[List[str]] = None + account_id: Optional[str] = ACCOUNT_ID, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + fields: Optional[List[str]] = None, ) -> RequestBuilder: since = since.strftime(DATE_FORMAT) if since else START_DATE[:10] until = until.strftime(DATE_FORMAT) if until else END_DATE[:10] @@ -174,7 +177,7 @@ def _job_start_request( "PENDING_BILLING_INFO", "PENDING_REVIEW", "PREAPPROVED", - "WITH_ISSUES" + "WITH_ISSUES", ], }, ], @@ -250,7 +253,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -434,7 +437,10 @@ def test_given_status_500_reduce_amount_of_data_when_read_then_limit_reduced(sel class TestIncremental(TestCase): @staticmethod def _read( - config_: ConfigBuilder, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False, json_schema: Optional[Dict[str, any]] = None + config_: ConfigBuilder, + state: Optional[List[AirbyteStateMessage]] = None, + expecting_exception: bool = False, + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: return read_output( config_builder=config_, @@ -442,7 +448,7 @@ def _read( sync_mode=SyncMode.incremental, state=state, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -467,7 +473,9 @@ def test_when_read_then_state_message_produced_and_state_match_start_interval(se ) output = self._read(config().with_account_ids([account_id]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_message == start_date.strftime(DATE_FORMAT) @@ -511,8 +519,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) expected_cursor_value = start_date.strftime(DATE_FORMAT) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_account_1 == expected_cursor_value diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index d437566f1d48c..16c2a1cfae36d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -96,7 +96,7 @@ def _read(config_: ConfigBuilder, expecting_exception: bool = False, json_schema stream_name=_STREAM_NAME, sync_mode=SyncMode.full_refresh, expecting_exception=expecting_exception, - json_schema=json_schema + json_schema=json_schema, ) @HttpMocker() @@ -244,7 +244,9 @@ def test_when_read_then_state_message_produced_and_state_match_latest_record(sel ) output = self._read(config().with_account_ids([account_id])) - cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_message == max_cursor_value @HttpMocker() @@ -276,8 +278,12 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2])) - cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + ) + cursor_value_from_state_account_2 = ( + AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) + ) assert cursor_value_from_state_account_1 == max_cursor_value_account_id_1 assert cursor_value_from_state_account_2 == max_cursor_value_account_id_2 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py index 2686186e18407..40278d92f44fd 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py @@ -34,7 +34,7 @@ def read_output( sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: Optional[bool] = False, - json_schema: Optional[Dict[str, any]] = None + json_schema: Optional[Dict[str, any]] = None, ) -> EntrypointOutput: _catalog = catalog(stream_name, sync_mode, json_schema) _config = config_builder.build() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 9b9a0e2a1f9fe..6e8b2d6f08ab3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -99,14 +99,10 @@ def test_init_statuses(self, api, some_config): end_date=datetime(2011, 1, 1), insights_lookback_window=28, fields=["account_id", "account_currency"], - filter_statuses=["ACTIVE", "ARCHIVED"] + filter_statuses=["ACTIVE", "ARCHIVED"], ) - assert stream.request_params()["filtering"] == [ - {'field': 'ad.effective_status', - 'operator': 'IN', - 'value': ['ACTIVE', 'ARCHIVED']} - ] + assert stream.request_params()["filtering"] == [{"field": "ad.effective_status", "operator": "IN", "value": ["ACTIVE", "ARCHIVED"]}] def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock @@ -464,7 +460,9 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) - assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1, "should be 37 slices because we ignore slices which are within insights_lookback_period" + assert ( + len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1 + ), "should be 37 slices because we ignore slices which are within insights_lookback_period" assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1) @@ -602,61 +600,78 @@ def test_start_date_with_lookback_window( @pytest.mark.parametrize( "breakdowns, record, expected_record", ( - ( - ["body_asset", ], - {"body_asset": {"id": "871246182", "text": "Some text"}}, - {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"} - ), - ( - ["call_to_action_asset",], - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, - {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"} - ), - ( - ["description_asset", ], - {"description_asset": {"id": "871246182", "text": "Some text"}}, - {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"} - ), - ( - ["image_asset", ], - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, - {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"} - ), - ( - ["link_url_asset", ], - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, - {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"} - ), - ( - ["title_asset", ], - {"title_asset": {"id": "871246182", "text": "Some text"}}, - {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"} - ), - ( - ["video_asset", ], - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - } - }, - { - "video_asset": { - "id": "871246182", "video_id": "video_id", "url": "url", - "thumbnail_url": "thumbnail_url", "video_name": "video_name" - }, - "video_asset_id": "871246182" - } - ), - ( - ["body_asset", "country"], - {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, - { - "body_asset": {"id": "871246182", "text": "Some text"}, - "country": "country", "dma": "dma", "body_asset_id": "871246182" - } - ), - ) + ( + [ + "body_asset", + ], + {"body_asset": {"id": "871246182", "text": "Some text"}}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "body_asset_id": "871246182"}, + ), + ( + [ + "call_to_action_asset", + ], + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}}, + {"call_to_action_asset": {"id": "871246182", "name": "Some name"}, "call_to_action_asset_id": "871246182"}, + ), + ( + [ + "description_asset", + ], + {"description_asset": {"id": "871246182", "text": "Some text"}}, + {"description_asset": {"id": "871246182", "text": "Some text"}, "description_asset_id": "871246182"}, + ), + ( + [ + "image_asset", + ], + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}}, + {"image_asset": {"id": "871246182", "hash": "hash", "url": "url"}, "image_asset_id": "871246182"}, + ), + ( + [ + "link_url_asset", + ], + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}}, + {"link_url_asset": {"id": "871246182", "website_url": "website_url"}, "link_url_asset_id": "871246182"}, + ), + ( + [ + "title_asset", + ], + {"title_asset": {"id": "871246182", "text": "Some text"}}, + {"title_asset": {"id": "871246182", "text": "Some text"}, "title_asset_id": "871246182"}, + ), + ( + [ + "video_asset", + ], + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + } + }, + { + "video_asset": { + "id": "871246182", + "video_id": "video_id", + "url": "url", + "thumbnail_url": "thumbnail_url", + "video_name": "video_name", + }, + "video_asset_id": "871246182", + }, + ), + ( + ["body_asset", "country"], + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma"}, + {"body_asset": {"id": "871246182", "text": "Some text"}, "country": "country", "dma": "dma", "body_asset_id": "871246182"}, + ), + ), ) def test_transform_breakdowns(self, api, some_config, breakdowns, record, expected_record): start_date = pendulum.parse("2024-01-01") @@ -674,36 +689,19 @@ def test_transform_breakdowns(self, api, some_config, breakdowns, record, expect @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - None, - ["date_start", "account_id", "ad_id"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + (None, ["date_start", "account_id", "ad_id"]), + ), ) def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -714,43 +712,38 @@ def test_primary_keys(self, api, some_config, breakdowns, expect_pks): start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) assert stream.primary_key == expect_pks @pytest.mark.parametrize( "breakdowns, expect_pks", ( - ( - ["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"] - ), - ( - ["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"] - ), - ( - ["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"] - ), - ( - ["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"] - ), - ( - ["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"] - ), - ( - ["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"] - ), - ( - ["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"] - ), - ( - ["video_asset", "skan_conversion_id", "place_page_id"], - ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"] - ), - ( - ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], - ["date_start", "account_id", "ad_id", "video_asset_id", "link_url_asset_id", "skan_conversion_id", "place_page_id", "gender"] - ), - ) + (["body_asset"], ["date_start", "account_id", "ad_id", "body_asset_id"]), + (["call_to_action_asset"], ["date_start", "account_id", "ad_id", "call_to_action_asset_id"]), + (["description_asset"], ["date_start", "account_id", "ad_id", "description_asset_id"]), + (["image_asset"], ["date_start", "account_id", "ad_id", "image_asset_id"]), + (["link_url_asset"], ["date_start", "account_id", "ad_id", "link_url_asset_id"]), + (["title_asset"], ["date_start", "account_id", "ad_id", "title_asset_id"]), + (["video_asset"], ["date_start", "account_id", "ad_id", "video_asset_id"]), + ( + ["video_asset", "skan_conversion_id", "place_page_id"], + ["date_start", "account_id", "ad_id", "video_asset_id", "skan_conversion_id", "place_page_id"], + ), + ( + ["video_asset", "link_url_asset", "skan_conversion_id", "place_page_id", "gender"], + [ + "date_start", + "account_id", + "ad_id", + "video_asset_id", + "link_url_asset_id", + "skan_conversion_id", + "place_page_id", + "gender", + ], + ), + ), ) def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pks): start_date = pendulum.parse("2024-01-01") @@ -761,7 +754,7 @@ def test_object_pk_added_to_schema(self, api, some_config, breakdowns, expect_pk start_date=start_date, end_date=end_date, insights_lookback_window=1, - breakdowns=breakdowns + breakdowns=breakdowns, ) schema = stream.get_json_schema() assert schema diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 885560c92d3a9..36d7d0c022235 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -105,7 +105,9 @@ def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, except FacebookRequestError: pytest.fail("Call rate error has not being handled") - def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception(self, requests_mock, api, fb_call_rate_response, account_id, some_config): + def test_given_rate_limit_reached_when_read_then_raise_transient_traced_exception( + self, requests_mock, api, fb_call_rate_response, account_id, some_config + ): requests_mock.register_uri( "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 68229d5923a1a..a18f3a3a95764 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -168,6 +168,7 @@ def test_should_not_migrate_upgraded_config(self): migration_instance = MigrateIncludeDeletedToStatusFilters() assert not migration_instance.should_migrate(new_config) + class TestMigrateSecretsPathInConnector: OLD_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_old_access_token_config.json") NEW_TEST_CONFIG_PATH_ACCESS_TOKEN = _config_path(f"{_SECRETS_TO_CREDENTIALS_CONFIGS_PATH}/test_new_access_token_config.json") @@ -178,7 +179,7 @@ class TestMigrateSecretsPathInConnector: def revert_migration(config_path: str) -> None: with open(config_path, "r") as test_config: config = json.load(test_config) - credentials = config.pop("credentials",{}) + credentials = config.pop("credentials", {}) credentials.pop("auth_type", None) with open(config_path, "w") as updated_config: config = json.dumps({**config, **credentials}) @@ -202,7 +203,7 @@ def test_migrate_access_token_config(self): assert original_config["access_token"] == credentials["access_token"] # revert the test_config to the starting point self.revert_migration(self.OLD_TEST_CONFIG_PATH_ACCESS_TOKEN) - + def test_migrate_client_config(self): migration_instance = MigrateSecretsPathInConnector() original_config = load_config(self.OLD_TEST_CONFIG_PATH_CLIENT) @@ -228,7 +229,7 @@ def test_should_not_migrate_new_client_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_CLIENT) migration_instance = MigrateSecretsPathInConnector() assert not migration_instance._should_migrate(new_config) - + def test_should_not_migrate_new_access_token_config(self): new_config = load_config(self.NEW_TEST_CONFIG_PATH_ACCESS_TOKEN) migration_instance = MigrateSecretsPathInConnector() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py index b593e7e1b1f3c..f51acdb1621e3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -113,7 +113,7 @@ "code": 100, } }, - } + }, # Error randomly happens for different connections. # Can be reproduced on https://developers.facebook.com/tools/explorer/?method=GET&path=act_&version=v17.0 # 1st reason: incorrect ad account id is used @@ -183,7 +183,7 @@ "error_user_msg": "profile should always be linked to delegate page", } }, - } + }, # Error happens on Video stream: https://graph.facebook.com/v17.0/act_XXXXXXXXXXXXXXXX/advideos # Recommendations says that the problem can be fixed by switching to Business Ad Account Id ), @@ -215,8 +215,8 @@ "message": "(#3018) The start date of the time range cannot be beyond 37 months from the current date", "type": "OAuthException", "code": 3018, - "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P" - } + "fbtrace_id": "Ag-P22y80OSEXM4qsGk2T9P", + } }, }, ), @@ -252,27 +252,28 @@ SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME = "error_400_service_temporarily_unavailable" SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE = { - "status_code": 503, - "json": { - "error": { - "message": "(#2) Service temporarily unavailable", - "type": "OAuthException", - "is_transient": True, - "code": 2, - "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", - } - }, - } + "status_code": 503, + "json": { + "error": { + "message": "(#2) Service temporarily unavailable", + "type": "OAuthException", + "is_transient": True, + "code": 2, + "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", + } + }, +} REDUCE_FIELDS_ERROR_TEST_NAME = "error_500_reduce_the_amount_of_data" REDUCE_FIELDS_ERROR_RESPONSE = { - "status_code": 500, - "json": { - "error": { - "message": "Please reduce the amount of data you're asking for, then retry your request", - "code": 1, - } - }, - } + "status_code": 500, + "json": { + "error": { + "message": "Please reduce the amount of data you're asking for, then retry your request", + "code": 1, + } + }, +} + class TestRealErrors: @pytest.mark.parametrize( @@ -406,23 +407,24 @@ def test_config_error_during_actual_nodes_read(self, requests_mock, name, friend assert error.failure_type == FailureType.config_error assert friendly_msg in error.message - @pytest.mark.parametrize("name, friendly_msg, config_error_response, failure_type", - [ - ( - REDUCE_FIELDS_ERROR_TEST_NAME, - "Please reduce the number of fields requested. Go to the schema tab, " - "select your source, and unselect the fields you do not need.", - REDUCE_FIELDS_ERROR_RESPONSE, - FailureType.config_error - ), - ( - SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, - "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", - SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, - FailureType.transient_error - ) - ] - ) + @pytest.mark.parametrize( + "name, friendly_msg, config_error_response, failure_type", + [ + ( + REDUCE_FIELDS_ERROR_TEST_NAME, + "Please reduce the number of fields requested. Go to the schema tab, " + "select your source, and unselect the fields you do not need.", + REDUCE_FIELDS_ERROR_RESPONSE, + FailureType.config_error, + ), + ( + SERVICE_TEMPORARILY_UNAVAILABLE_TEST_NAME, + "The Facebook API service is temporarily unavailable. This issue should resolve itself, and does not require further action.", + SERVICE_TEMPORARILY_UNAVAILABLE_RESPONSE, + FailureType.transient_error, + ), + ], + ) def test_config_error_that_was_retried_when_reading_nodes(self, requests_mock, name, friendly_msg, config_error_response, failure_type): """This test covers errors that have been resolved in the past with a retry strategy, but it could also can fail after retries, then, we need to provide the user with a humanized error explaining what just happened""" @@ -498,7 +500,7 @@ def test_config_error_insights_during_actual_nodes_read(self, requests_mock, nam assert friendly_msg in error.message def test_retry_for_cannot_include_error(self, requests_mock): - """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868 """ + """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868""" api = API(access_token=some_config["access_token"], page_size=100) stream = AdsInsights( @@ -516,7 +518,7 @@ def test_retry_for_cannot_include_error(self, requests_mock): "error": { "message": "(#100) Cannot include video_avg_time_watched_actions, video_continuous_2_sec_watched_actions in summary param because they weren't there while creating the report run.", "type": "OAuthException", - "code": 100 + "code": 100, } }, } @@ -594,24 +596,22 @@ def test_traced_exception_with_api_error(): request_context={}, http_status=400, http_headers={}, - body='{"error": {"message": "Error validating access token", "code": 190}}' + body='{"error": {"message": "Error validating access token", "code": 190}}', ) error.api_error_message = MagicMock(return_value="Error validating access token") - + result = traced_exception(error) - + assert isinstance(result, AirbyteTracedException) - assert result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + assert ( + result.message == "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + ) assert result.failure_type == FailureType.config_error def test_traced_exception_without_api_error(): error = FacebookRequestError( - message="Call was unsuccessful. The Facebook API has imploded", - request_context={}, - http_status=408, - http_headers={}, - body='{}' + message="Call was unsuccessful. The Facebook API has imploded", request_context={}, http_status=408, http_headers={}, body="{}" ) error.api_error_message = MagicMock(return_value=None) diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py index 36fe896ca5d84..8f553d3d745ce 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py @@ -9,9 +9,10 @@ def test_field_transformation(): - with open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, open( - f"{os.path.dirname(__file__)}/transformed_record.json", "r" - ) as transformed_record: + with ( + open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, + open(f"{os.path.dirname(__file__)}/transformed_record.json", "r") as transformed_record, + ): initial_record = json.loads(initial_record.read()) transformed_record = json.loads(transformed_record.read()) record_transformation = CustomFieldTransformation(config={}, parameters={"name": "page"}) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py index cb8041a4f3967..4e0ac799cc8d4 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -84,7 +84,8 @@ def is_ssh_ready(ip, port): @pytest.fixture(scope="session") def move_sample_files_to_tmp(): """Copy sample files to /tmp so that they can be accessed by the dockerd service in the context of Dagger test runs. - The sample files are mounted to the SSH service from the container under test (container) following instructions of docker-compose.yml in this directory.""" + The sample files are mounted to the SSH service from the container under test (container) following instructions of docker-compose.yml in this directory. + """ sample_files = Path(HERE / "sample_files") shutil.copytree(sample_files, "/tmp/s3_sample_files") yield True diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index fc28d7e66c558..ccab3fcfd09bc 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -99,8 +99,7 @@ def test_load_dataframes_xlsx(config, absolute_path, test_files, file_name, shou assert read_file.equals(expected) -@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), - ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) +@pytest.mark.parametrize("file_format, file_path", [("json", "formats/json/demo.json"), ("jsonl", "formats/jsonl/jsonl_nested.jsonl")]) def test_load_nested_json(client, config, absolute_path, test_files, file_format, file_path): if file_format == "jsonl": config["format"] = file_format @@ -131,7 +130,8 @@ def test_cache_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv" with open(f, mode="rb") as file: assert client._cache_stream(file) - + + def test_unzip_stream(client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/test.csv.zip" with open(f, mode="rb") as file: @@ -223,10 +223,11 @@ def patched_open(self): def test_backoff_handler(caplog): details = {"tries": 1, "wait": 1} backoff_handler(details) - expected = [('airbyte', 20, 'Caught retryable error after 1 tries. Waiting 1 seconds then retrying...')] + expected = [("airbyte", 20, "Caught retryable error after 1 tries. Waiting 1 seconds then retrying...")] assert caplog.record_tuples == expected + def generate_excel_file(data): """ Helper function to generate an Excel file with the given data. diff --git a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py index 2e273232b3237..2c75ac63620ea 100644 --- a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py +++ b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py @@ -33,6 +33,7 @@ def config(request): } return args + @fixture() def legacy_config(request): args = { @@ -44,6 +45,7 @@ def legacy_config(request): } return args + @fixture() def config_no_engine(): args = { @@ -109,12 +111,14 @@ def test_parse_config(config, logger): result = parse_config(config, logger) assert result["engine_url"] == "override_engine.api.firebolt.io" + def test_parse_legacy_config(legacy_config, logger): result = parse_config(legacy_config, logger) assert result["database"] == "my_database" assert result["auth"].username == "my@username" assert result["auth"].password == "my_password" + @patch("source_firebolt.database.connect") def test_connection(mock_connection, config, config_no_engine, logger): establish_connection(config, logger) diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py index 1fe90e1f13746..9d71099135bbc 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/conftest.py @@ -77,4 +77,3 @@ def connector_setup_fixture(docker_client) -> None: container.kill() container.remove() - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py index 105a55e1ee085..aea4bf84db9b3 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/conftest.py @@ -63,9 +63,10 @@ def zip_file(): uri=str(Path(__file__).parent / "resource/files/test.csv.zip"), last_modified=datetime.today(), mime_type=".zip", - displayed_uri="resource/files/test.csv.zip" + displayed_uri="resource/files/test.csv.zip", ) + @pytest.fixture def mocked_blob(): blob = Mock() diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py index ea301e8d9e12c..eab24e5e8d145 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config.py @@ -7,4 +7,3 @@ def test_documentation_url(): assert "https" in Config.documentation_url() - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py index 142a7f8939c8e..6fe9231f0b0b7 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_config_migrations.py @@ -15,24 +15,32 @@ def load_config(path: str) -> Mapping[str, Any]: with open(path, "r") as f: return json.load(f) + def revert_config(path: str) -> None: migrated_config = load_config(path) del migrated_config["credentials"] with open(path, "w") as f: f.write(json.dumps(migrated_config)) + @pytest.mark.parametrize( "config_file_path, run_revert", [ # Migration is required (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_config.json"), True), # New config format - (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False) - ] + (str(pathlib.Path(__file__).parent / "resource/config_migrations/service_account_with_credentials_config.json"), False), + ], ) def test_migrate_config(config_file_path, run_revert): args = ["check", "--config", config_file_path] - source = SourceGCS(MagicMock(), MagicMock, None, AirbyteEntrypoint.extract_config(args), None,) + source = SourceGCS( + MagicMock(), + MagicMock, + None, + AirbyteEntrypoint.extract_config(args), + None, + ) MigrateServiceAccount().migrate(args, source) migrated_config = load_config(config_file_path) @@ -43,4 +51,3 @@ def test_migrate_config(config_file_path, run_revert): if run_revert: revert_config(config_file_path) - diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py index 59095eead97d8..3fdd41387a330 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_cursor.py @@ -125,9 +125,6 @@ def test_add_file_zip_files(mocked_reader, zip_file, logger): cursor = Cursor(stream_config=FileBasedStreamConfig(name="test", globs=["**/*.zip"], format={"filetype": "csv"})) cursor.add_file(zip_file) - saved_history_cursor = datetime.strptime( - cursor._file_to_datetime_history[zip_file.displayed_uri], - cursor.DATE_TIME_FORMAT - ) + saved_history_cursor = datetime.strptime(cursor._file_to_datetime_history[zip_file.displayed_uri], cursor.DATE_TIME_FORMAT) assert saved_history_cursor == zip_file.last_modified diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py index 163aaab29bf17..9ef22105964b9 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream.py @@ -9,8 +9,15 @@ def test_transform_record(zip_file, mocked_reader, logger): stream = GCSStream( - config=Mock(), catalog_schema=Mock(), stream_reader=Mock(), availability_strategy=Mock(), discovery_policy=Mock(),parsers=Mock(), - validation_policy=Mock(), errors_collector=Mock(), cursor=Mock() + config=Mock(), + catalog_schema=Mock(), + stream_reader=Mock(), + availability_strategy=Mock(), + discovery_policy=Mock(), + parsers=Mock(), + validation_policy=Mock(), + errors_collector=Mock(), + cursor=Mock(), ) last_updated = zip_file.last_modified.isoformat() transformed_record = stream.transform_record({"field1": 1}, zip_file, last_updated) @@ -19,11 +26,7 @@ def test_transform_record(zip_file, mocked_reader, logger): assert transformed_record["_ab_source_file_url"] != zip_file.uri last_updated = datetime.today().isoformat() - csv_file = GCSRemoteFile( - uri="https://storage.googleapis.com/test/test", - last_modified=last_updated, - mime_type = "csv" - ) + csv_file = GCSRemoteFile(uri="https://storage.googleapis.com/test/test", last_modified=last_updated, mime_type="csv") transformed_record = stream.transform_record({"field1": 1}, csv_file, last_updated) assert transformed_record["_ab_source_file_url"] == csv_file.uri diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py index b5d5b501872e4..7e645ac95769b 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_stream_reader.py @@ -13,10 +13,7 @@ def test_get_matching_files_with_no_prefix(logger, mocked_reader): mocked_reader._config = Config( - credentials=ServiceAccountCredentials( - service_account='{"type": "service_account"}', - auth_type="Service" - ), + credentials=ServiceAccountCredentials(service_account='{"type": "service_account"}', auth_type="Service"), bucket="test_bucket", streams=[], ) diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py index d4ea6e2fb08f9..f2595830b6db9 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/test_zip_helper.py @@ -6,7 +6,6 @@ def test_get_gcs_remote_files(mocked_blob, zip_file, caplog): - files = list(ZipHelper(mocked_blob, zip_file,tempfile.TemporaryDirectory()).get_gcs_remote_files()) + files = list(ZipHelper(mocked_blob, zip_file, tempfile.TemporaryDirectory()).get_gcs_remote_files()) assert len(files) == 1 assert "Picking up file test.csv from zip archive" in caplog.text - diff --git a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py index 79ec73a9cd2f5..73c5ceca7e601 100644 --- a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py +++ b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py @@ -30,13 +30,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: diff --git a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py index bde678baef18a..d0ff53407c046 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/integration/test_assignees.py @@ -162,12 +162,19 @@ def test_read_full_refresh_emits_per_partition_state(self): per_partition_state_1 = {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}} per_partition_state_2 = {"partition": {"repository": "airbytehq/mock-test-2"}, "cursor": {"__ab_full_refresh_sync_complete": True}} - incoming_state = StateBuilder().with_stream_state("assignees", { - "states": [ - {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, - ] - }).build() + incoming_state = ( + StateBuilder() + .with_stream_state( + "assignees", + { + "states": [ + {"partition": {"repository": "airbytehq/mock-test-0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"repository": "airbytehq/mock-test-1"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + ] + }, + ) + .build() + ) source = SourceGithub() actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), state=incoming_state) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index d8944137c766f..32f0952c8c08e 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -102,10 +102,34 @@ def test_backoff_time(time_mock, http_status, response_headers, expected_backoff @pytest.mark.parametrize( ("http_status", "response_headers", "text", "response_action", "error_message"), [ - (HTTPStatus.OK, {"X-RateLimit-Resource": "graphql"}, '{"errors": [{"type": "RATE_LIMITED"}]}', ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.OK}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"X-RateLimit-Remaining": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "0"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), - (HTTPStatus.FORBIDDEN, {"Retry-After": "60"}, "", ResponseAction.RATE_LIMITED, f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying..."), + ( + HTTPStatus.OK, + {"X-RateLimit-Resource": "graphql"}, + '{"errors": [{"type": "RATE_LIMITED"}]}', + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.OK}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"X-RateLimit-Remaining": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "0"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), + ( + HTTPStatus.FORBIDDEN, + {"Retry-After": "60"}, + "", + ResponseAction.RATE_LIMITED, + f"Response status code: {HTTPStatus.FORBIDDEN}. Retrying...", + ), (HTTPStatus.INTERNAL_SERVER_ERROR, {}, "", ResponseAction.RETRY, "Internal server error."), (HTTPStatus.BAD_GATEWAY, {}, "", ResponseAction.RETRY, "Bad gateway."), (HTTPStatus.SERVICE_UNAVAILABLE, {}, "", ResponseAction.RETRY, "Service unavailable."), @@ -451,18 +475,18 @@ def test_stream_commits_incremental_read(): "name": "branch", "commit": { "sha": "74445338726f0f8e1c27c10dce90ca00c5ae2858", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/74445338726f0f8e1c27c10dce90ca00c5ae2858", }, - "protected": False + "protected": False, }, { "name": "main", "commit": { "sha": "c27c10dce90ca00c5ae285874445338726f0f8e1", - "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1" + "url": "https://api.github.com/repos/airbytehq/airbyte/commits/c27c10dce90ca00c5ae285874445338726f0f8e1", }, - "protected": False - } + "protected": False, + }, ], status=200, ) @@ -1361,11 +1385,7 @@ def test_stream_projects_v2_graphql_retry(time_mock, rate_limit_mock_response): } stream = ProjectsV2(**repository_args_with_start_date) resp = responses.add( - responses.POST, - "https://api.github.com/graphql", - json={"errors": "not found"}, - status=200, - headers={'Retry-After': '5'} + responses.POST, "https://api.github.com/graphql", json={"errors": "not found"}, status=200, headers={"Retry-After": "5"} ) backoff_strategy = GithubStreamABCBackoffStrategy(stream) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py index ec47f547f591c..12b176b266aee 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py @@ -30,13 +30,11 @@ class MigrateStringToArray(ABC): @property @abc.abstractmethod - def migrate_from_key(self) -> str: - ... + def migrate_from_key(self) -> str: ... @property @abc.abstractmethod - def migrate_to_key(self) -> str: - ... + def migrate_to_key(self) -> str: ... @classmethod def _should_migrate(cls, config: Mapping[str, Any]) -> bool: diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py index 4e7e0b40483fc..5b1c927a258fb 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_partition_routers.py @@ -56,9 +56,7 @@ def test_projects_stream_slices_with_group_project_ids(self, requests_mock): url=f"https://gitlab.com/api/v4/groups/{group_id}?per_page=50", json=[{"id": group_id, "projects": [{"id": project_id, "path_with_namespace": project_id}]}], ) - expected_stream_slices.append( - StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={}) - ) + expected_stream_slices.append(StreamSlice(partition={"id": project_id.replace("/", "%2F")}, cursor_slice={})) requests_mock.get(url=GROUPS_LIST_URL, json=groups_list_response) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index b650c9e52b6d8..947f572341734 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -19,9 +19,7 @@ def test_streams(config): def test_connection_success(config, requests_mock): requests_mock.get(url="/api/v4/groups", json=[{"id": "g1"}]) - requests_mock.get( - url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}] - ) + requests_mock.get(url="/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]) requests_mock.get(url="/api/v4/projects/p1", json={"id": "p1"}) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) @@ -30,9 +28,7 @@ def test_connection_success(config, requests_mock): def test_connection_invalid_projects_and_projects(config_with_project_groups, requests_mock): requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1?per_page=50", status_code=404) - requests_mock.register_uri( - "GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404 - ) + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404) requests_mock.register_uri("GET", "https://gitlab.com/api/v4/projects/p1?per_page=50&statistics=1", status_code=404) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config_with_project_groups) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index 9eef564ef235b..cf0dbbb4091a7 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -123,7 +123,7 @@ def test_should_retry(requests_mock, stream_name, extra_mocks): "user_full_name": "John", "environment_name": "dev", "project_id": "p_1", - } + }, ), ( "merge_request_commits", @@ -157,11 +157,12 @@ def test_stream_slices_child_stream(requests_mock): url="https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1", json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}], ) - stream_state = {"13082000": {""'created_at': "2021-03-10T23:58:1213"}} + stream_state = {"13082000": {"" "created_at": "2021-03-10T23:58:1213"}} slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state=stream_state)) assert slices + def test_request_params(): commits = get_stream_by_name("commits", CONFIG) assert commits.retriever.requester.get_request_params() == {"with_stats": "true"} diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py index 5019d87b7b44c..a347efae2f82c 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py @@ -11,7 +11,7 @@ ("http://example", (True, "http", "example")), ("test://example.com", (False, "", "")), ("https://example.com/test/test2", (False, "", "")), - ) + ), ) def test_parse_url(url, expected): assert parse_url(url) == expected diff --git a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py index f2b023043a0b8..822bd1a8523b5 100644 --- a/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py +++ b/airbyte-integrations/connectors/source-gong/unit_tests/test_request_with_filter.py @@ -20,32 +20,28 @@ class TestGetRequestFilterOptions(unittest.TestCase): def setUp(self): self.instance = IncrementalSingleBodyFilterCursor( - start_datetime= MinMaxDatetime(datetime= "2024-03-25", datetime_format= "%Y-%m-%dT%H:%M:%SZ", parameters = None), - cursor_field = "reviewTime", - datetime_format= "%Y-%m-%d", - config= config, - parameters = None + start_datetime=MinMaxDatetime(datetime="2024-03-25", datetime_format="%Y-%m-%dT%H:%M:%SZ", parameters=None), + cursor_field="reviewTime", + datetime_format="%Y-%m-%d", + config=config, + parameters=None, ) - + def test_get_request_filter_options_no_stream_slice(self): expected = {} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, None) assert result == expected - + def test_get_request_filter_options_with_start_time_option(self): - expected = {'filter': {'reviewFromDate': '2024-03-25'}} - - self.instance.start_time_option = RequestOption( - inject_into = "body_json", - field_name = "filter, reviewFromDate", - parameters = None - ) - self.instance.stream_slice = {'start_time': '2024-03-25', 'end_time': '2024-03-29'} + expected = {"filter": {"reviewFromDate": "2024-03-25"}} + + self.instance.start_time_option = RequestOption(inject_into="body_json", field_name="filter, reviewFromDate", parameters=None) + self.instance.stream_slice = {"start_time": "2024-03-25", "end_time": "2024-03-29"} option_type = "body_json" result = self.instance._get_request_filter_options(option_type, self.instance.stream_slice) assert result == expected -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py index f409e0f1ac52a..21623048feda0 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -55,6 +55,7 @@ def mock_oauth_call(requests_mock): def customers(config): return [CustomerModel(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + @pytest.fixture def additional_customers(config, customers): return customers + [CustomerModel(id="789", time_zone="local", is_manager_account=False)] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py index 49679c3be829b..f8eb989c8a66e 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -35,7 +35,10 @@ def mock_get_customers(mocker): "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", ), (["QUERY_ERROR"], "Incorrect custom query. Error in query: unexpected end of query."), - (["UNRECOGNIZED_FIELD"], "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid."), + ( + ["UNRECOGNIZED_FIELD"], + "The Custom Query: `None` has unrecognized field in the query. Please make sure the field exists or name entered is valid.", + ), ( ["RESOURCE_EXHAUSTED"], ( diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index e3bec07f29762..f38143b2cf142 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -148,11 +148,12 @@ def test_get_field_value(): response = GoogleAds.get_field_value(MockedDateSegment(date), field, {}) assert response == date + def test_get_field_value_object(): expected_response = [ - { "text": "An exciting headline", "policySummaryInfo": { "reviewStatus": "REVIEWED","approvalStatus":"APPROVED" } }, - { "text": "second" }, - ] + {"text": "An exciting headline", "policySummaryInfo": {"reviewStatus": "REVIEWED", "approvalStatus": "APPROVED"}}, + {"text": "second"}, + ] field = "ad_group_ad.ad.responsive_search_ad.headlines" ads_row = GoogleAdsRow( ad_group_ad={ @@ -161,14 +162,9 @@ def test_get_field_value_object(): "headlines": [ { "text": "An exciting headline", - "policy_summary_info": { - "review_status": "REVIEWED", - "approval_status": "APPROVED" - } + "policy_summary_info": {"review_status": "REVIEWED", "approval_status": "APPROVED"}, }, - { - "text": "second" - } + {"text": "second"}, ] } } @@ -176,13 +172,14 @@ def test_get_field_value_object(): ) response = GoogleAds.get_field_value(ads_row, field, {}) - assert [json.loads(i) for i in response] == expected_response + assert [json.loads(i) for i in response] == expected_response + def test_get_field_value_strings(): expected_response = [ - "http://url_one.com", - "https://url_two.com", - ] + "http://url_one.com", + "https://url_two.com", + ] ads_row = GoogleAdsRow( ad_group_ad={ "ad": { @@ -197,6 +194,7 @@ def test_get_field_value_strings(): response = GoogleAds.get_field_value(ads_row, field, {}) assert response == expected_response + def test_parse_single_result(): date = "2001-01-01" response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py index 34b89d1e2e881..ec7eeadd02b19 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py @@ -82,7 +82,7 @@ def test_change_status_stream(config, customers): def test_change_status_stream_slices(config, additional_customers): - """ Change status stream slices should return correct empty slices for the new customers """ + """Change status stream slices should return correct empty slices for the new customers""" google_api = MockGoogleAds(credentials=config["credentials"]) stream = ChangeStatus(api=google_api, customers=additional_customers) @@ -94,12 +94,19 @@ def test_change_status_stream_slices(config, additional_customers): result_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(result_slices) == 2 - assert result_slices == [{'start_date': '2023-11-01 12:36:04.772447', 'end_date': '2023-11-02 00:00:00.000000', 'customer_id': '123', - 'login_customer_id': None}, {'customer_id': '789', 'login_customer_id': None}] + assert result_slices == [ + { + "start_date": "2023-11-01 12:36:04.772447", + "end_date": "2023-11-02 00:00:00.000000", + "customer_id": "123", + "login_customer_id": None, + }, + {"customer_id": "789", "login_customer_id": None}, + ] def test_incremental_events_stream_slices(config, additional_customers): - """ Test if the empty slice will be produced for the new customers """ + """Test if the empty slice will be produced for the new customers""" stream_state = {"change_status": {"123": {"change_status.last_change_date_time": "2023-06-12 13:20:01.003295"}}} google_api = MockGoogleAds(credentials=config["credentials"]) @@ -121,12 +128,21 @@ def test_incremental_events_stream_slices(config, additional_customers): stream_slices = list(stream.stream_slices(stream_state=stream_state)) assert len(stream_slices) == 2 - assert stream_slices == [{'customer_id': '123', 'updated_ids': {'2', '1'}, 'deleted_ids': {'3', '4'}, - 'record_changed_time_map': {'1': '2023-06-13 12:36:01.772447', '2': '2023-06-13 12:36:02.772447', - '3': '2023-06-13 12:36:03.772447', '4': '2023-06-13 12:36:04.772447'}, - 'login_customer_id': None}, - {'customer_id': '789', 'updated_ids': set(), 'deleted_ids': set(), 'record_changed_time_map': {}, - 'login_customer_id': None}] + assert stream_slices == [ + { + "customer_id": "123", + "updated_ids": {"2", "1"}, + "deleted_ids": {"3", "4"}, + "record_changed_time_map": { + "1": "2023-06-13 12:36:01.772447", + "2": "2023-06-13 12:36:02.772447", + "3": "2023-06-13 12:36:03.772447", + "4": "2023-06-13 12:36:04.772447", + }, + "login_customer_id": None, + }, + {"customer_id": "789", "updated_ids": set(), "deleted_ids": set(), "record_changed_time_map": {}, "login_customer_id": None}, + ] def test_child_incremental_events_read(config, customers): diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index d0665bea9ac92..a35c5098ca79f 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -251,8 +251,10 @@ def test_retry_500_raises_transient_error(mocker, config, customers): with pytest.raises(AirbyteTracedException) as exception: records = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) - assert exception.value.internal_message == ("Internal Error encountered Unable to fetch data from Google Ads API due to " - "temporal error on the Google Ads server. Please retry again later. ") + assert exception.value.internal_message == ( + "Internal Error encountered Unable to fetch data from Google Ads API due to " + "temporal error on the Google Ads server. Please retry again later. " + ) assert exception.value.failure_type == FailureType.transient_error assert mocked_search.call_count == 5 assert records == [] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py index bd5af212c081f..3148283084473 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py @@ -35,7 +35,7 @@ def test_check_initial_quota_is_empty(): "potentiallyThresholdedRequestsPerHour": {"consumed": 1, "remaining": 26}, } }, - False, # partial_quota + False, # partial_quota ResponseAction.RETRY, None, # backoff_time_exp False, # stop_iter_exp diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index e86201860e8bf..e7aa6e2e2c324 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -111,32 +111,42 @@ def test_check_failure_throws_exception(requests_mock, config_gen, error_code): def test_exhausted_quota_recovers_after_two_retries(requests_mock, config_gen): """ - If the account runs out of quota the api will return a message asking us to back off for one hour. - We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not - it will backoff again 30 minutes and quote should be reestablished by then. - Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota - recovery time and also the backoff time. + If the account runs out of quota the api will return a message asking us to back off for one hour. + We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not + it will backoff again 30 minutes and quote should be reestablished by then. + Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota + recovery time and also the backoff time. """ requests_mock.register_uri( "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} ) - error_response = {"error": {"message":"Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see"}} + error_response = { + "error": { + "message": "Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see" + } + } requests_mock.register_uri( "GET", "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata", # first try we get 429 t=~0 - [{"json": error_response, "status_code": 429}, - # first retry we get 429 t=~1800 - {"json": error_response, "status_code": 429}, - # second retry quota is recovered, t=~3600 - {"json": { - "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], - "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], - }, "status_code": 200} - ] + [ + {"json": error_response, "status_code": 429}, + # first retry we get 429 t=~1800 + {"json": error_response, "status_code": 429}, + # second retry quota is recovered, t=~3600 + { + "json": { + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + "status_code": 200, + }, + ], ) + def fix_time(time): - return int(time / 600 ) + return int(time / 600) + source = SourceGoogleAnalyticsDataApi() logger = MagicMock() max_time_fixed = fix_time(GoogleAnalyticsDatApiErrorHandler.QUOTA_RECOVERY_TIME) @@ -146,10 +156,18 @@ def fix_time(time): potentially_thresholded_requests_per_hour_mapping_fixed = { **potentially_thresholded_requests_per_hour_mapping, "backoff": fixed_threshold_backoff_time, - } + } with ( - patch.object(GoogleAnalyticsDatApiErrorHandler, 'QUOTA_RECOVERY_TIME', new=max_time_fixed), - patch.object(GoogleAnalyticsApiQuotaBase, 'quota_mapping', new={**GoogleAnalyticsApiQuotaBase.quota_mapping,"potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed})): + patch.object(GoogleAnalyticsDatApiErrorHandler, "QUOTA_RECOVERY_TIME", new=max_time_fixed), + patch.object( + GoogleAnalyticsApiQuotaBase, + "quota_mapping", + new={ + **GoogleAnalyticsApiQuotaBase.quota_mapping, + "potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed, + }, + ), + ): output = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert output == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) @@ -164,7 +182,7 @@ def test_check_failure(requests_mock, config_gen, error_code): ) source = SourceGoogleAnalyticsDataApi() logger = MagicMock() - with patch.object(HttpStatusErrorHandler, 'max_retries', new=0): + with patch.object(HttpStatusErrorHandler, "max_retries", new=0): airbyte_status = source.check(logger, config_gen(property_ids=["UA-11111111"])) assert airbyte_status.status == Status.FAILED assert airbyte_status.message == repr("Failed to get metadata, over quota, try later") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 3ab018bbe6d35..e44d6d25f1f1b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -85,10 +85,9 @@ def test_request_body_json(patch_base_class): {"name": "browser"}, ], "keepEmptyRows": True, - "dateRanges": [{ - "startDate": request_body_params["stream_slice"]["startDate"], - "endDate": request_body_params["stream_slice"]["endDate"] - }], + "dateRanges": [ + {"startDate": request_body_params["stream_slice"]["startDate"], "endDate": request_body_params["stream_slice"]["endDate"]} + ], "returnPropertyQuota": True, "offset": str(0), "limit": "100000", @@ -269,8 +268,8 @@ def test_should_retry(patch_base_class, http_status, response_action_expected, r if response_body: json_data = response_body response_mock._content = str.encode(json.dumps(json_data)) - response_mock.headers = {'Content-Type': 'application/json'} - response_mock.encoding = 'utf-8' + response_mock.headers = {"Content-Type": "application/json"} + response_mock.encoding = "utf-8" stream = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]) assert stream.get_error_handler().interpret_response(response_mock).response_action == response_action_expected @@ -312,6 +311,7 @@ def test_stream_slices(): {"startDate": "2022-12-30", "endDate": "2023-01-01"}, ] + @freeze_time("2023-01-01 00:00:00") def test_full_refresh(): """ @@ -319,9 +319,7 @@ def test_full_refresh(): """ config = {"date_ranges_start_date": datetime.date(2022, 12, 29), "window_in_days": 1, "dimensions": ["browser", "country", "language"]} stream = GoogleAnalyticsDataApiBaseStream(authenticator=None, config=config) - full_refresh_state = { - "__ab_full_refresh_state_message": True - } + full_refresh_state = {"__ab_full_refresh_state_message": True} slices = list(stream.stream_slices(sync_mode=None, stream_state=full_refresh_state)) assert slices == [ {"startDate": "2022-12-29", "endDate": "2022-12-29"}, @@ -423,47 +421,37 @@ def test_read_incremental(requests_mock): {"property_id": 123, "yearWeek": "202202", "totalUsers": 140, "startDate": "2022-01-10", "endDate": "2022-01-10"}, ] + @pytest.mark.parametrize( "config_dimensions, expected_state", [ pytest.param(["browser", "country", "language", "date"], {"date": "20240320"}, id="test_date_no_cursor_field_dimension"), pytest.param(["browser", "country", "language"], {}, id="test_date_cursor_field_dimension"), - ] + ], ) def test_get_updated_state(config_dimensions, expected_state): config = { - "credentials": { - "auth_type": "Service", - "credentials_json": "{ \"client_email\": \"a@gmail.com\", \"client_id\": \"1234\", \"client_secret\": \"5678\", \"private_key\": \"5678\"}" - }, - "date_ranges_start_date": "2023-04-01", - "window_in_days": 30, - "property_ids": ["123"], - "custom_reports_array": [ - { - "name": "pivot_report", - "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], - "dimensions": config_dimensions, - "metrics": ["sessions"], - "pivots": [ - { - "fieldNames": ["browser"], - "limit": 5 - }, - { - "fieldNames": ["country"], - "limit": 250 - }, + "credentials": { + "auth_type": "Service", + "credentials_json": '{ "client_email": "a@gmail.com", "client_id": "1234", "client_secret": "5678", "private_key": "5678"}', + }, + "date_ranges_start_date": "2023-04-01", + "window_in_days": 30, + "property_ids": ["123"], + "custom_reports_array": [ { - "fieldNames": ["language"], - "limit": 15 + "name": "pivot_report", + "dateRanges": [{"startDate": "2020-09-01", "endDate": "2020-09-15"}], + "dimensions": config_dimensions, + "metrics": ["sessions"], + "pivots": [ + {"fieldNames": ["browser"], "limit": 5}, + {"fieldNames": ["country"], "limit": 250}, + {"fieldNames": ["language"], "limit": 15}, + ], + "cohortSpec": {"enabled": "false"}, } - ], - "cohortSpec": { - "enabled": "false" - } - } - ] + ], } source = SourceGoogleAnalyticsDataApi() config = source._validate_and_transform(config, report_names=set()) diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py index 8dcb7e52e2236..2740a3d88ee5d 100644 --- a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py @@ -18,11 +18,11 @@ ("https://drive.google.com/drive/my-drive", None, True), ("http://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", None, True), ("https://drive.google.com/", None, True), - ] + ], ) def test_get_folder_id(input, output, raises): if raises: with pytest.raises(ValueError): get_folder_id(input) else: - assert get_folder_id(input) == output \ No newline at end of file + assert get_folder_id(input) == output diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 3c2b6dedfbe56..f8eeb1344f80b 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -133,7 +133,9 @@ def test_forbidden_should_retry(requests_mock, forbidden_error_message_json): def test_bad_aggregation_type_should_retry(requests_mock, bad_aggregation_type): stream = SearchAnalyticsKeywordSiteReportBySite(None, ["https://example.com"], "2021-01-01", "2021-01-02") - requests_mock.post(f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]}) + requests_mock.post( + f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]} + ) slice = list(stream.stream_slices(None))[0] url = stream.url_base + stream.path(None, slice) requests_mock.get(url, status_code=400, json=bad_aggregation_type) diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py index 7f1a4c699a8b6..56bf660e5bf89 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py @@ -28,10 +28,11 @@ def maker(spreadsheet_id, sheet_name): sheets=[ Sheet( data=[GridData(rowData=[RowData(values=[CellData(formattedValue="ID")])])], - properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}) + properties=SheetProperties(title=sheet_name, gridProperties={"rowCount": 2}), ), ], ) + return maker @@ -39,6 +40,7 @@ def maker(spreadsheet_id, sheet_name): def spreadsheet_values(): def maker(spreadsheet_id): return SpreadsheetValues(spreadsheetId=spreadsheet_id, valueRanges=[ValueRange(values=[["1"]])]) + return maker @@ -51,4 +53,5 @@ def maker(*name_schema_pairs): sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) + return maker diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py index d86d9fbaaa2bc..fc37aeebd8de3 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/custom_http_mocker.py @@ -20,14 +20,16 @@ def parse_and_transform(parse_result_str: str): # Convert the ParseResult string into a dictionary components = eval(f"dict({parse_result_part})") - url = urlunparse(( - components["scheme"], - components["netloc"], - components["path"], - components["params"], - components["query"], - components["fragment"], - )) + url = urlunparse( + ( + components["scheme"], + components["netloc"], + components["path"], + components["params"], + components["query"], + components["fragment"], + ) + ) return url @@ -40,6 +42,7 @@ class CustomHttpMocker: Note: there is only support for get and post method and url matching ignoring the body but this is enough for the current test set. """ + requests_mapper: Dict = {} def post(self, request: HttpRequest, response: HttpResponse): @@ -62,9 +65,9 @@ def mock_request(self, uri, method="GET", body=None, headers=None, **kwargs): return mocked_response # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` - def __call__(self, test_func): # type: ignore + def __call__(self, test_func): # type: ignore @wraps(test_func) - def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed + def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed kwargs["http_mocker"] = self with patch("httplib2.Http.request", side_effect=self.mock_request): diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py index af585b9e5fd44..222a31853b87c 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/request_builder.py @@ -8,13 +8,13 @@ GOOGLE_SHEETS_BASE_URL = "https://sheets.googleapis.com/v4/spreadsheets" OAUTH_AUTHORIZATION_ENDPOINT = "https://oauth2.googleapis.com" + class RequestBuilder: @classmethod def get_account_endpoint(cls) -> RequestBuilder: return cls(resource="values:batchGet") - - def __init__(self, resource:str=None) -> None: + def __init__(self, resource: str = None) -> None: self._spreadsheet_id = None self._query_params = {} self._body = None diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py index 4fdcd7fe0309d..959a854fcba38 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_credentials.py @@ -59,5 +59,5 @@ "auth_type": "Client", "client_id": "43987534895734985.apps.googleusercontent.com", "client_secret": "2347586435987643598", - "refresh_token": "1//4398574389537495437983457985437" - } + "refresh_token": "1//4398574389537495437983457985437", +} diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py index 6075f68535923..f14a4c1ac116e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/integration/test_source.py @@ -29,15 +29,9 @@ _C_STREAM_NAME = "c_stream_name" -_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": oauth_credentials -} +_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": oauth_credentials} -_SERVICE_CONFIG = { - "spreadsheet_id": _SPREADSHEET_ID, - "credentials": service_account_credentials -} +_SERVICE_CONFIG = {"spreadsheet_id": _SPREADSHEET_ID, "credentials": service_account_credentials} class GoogleSheetSourceTest(TestCase): @@ -49,14 +43,11 @@ def setUp(self) -> None: @staticmethod def authorize(http_mocker: HttpMocker): # Authorization request with user credentials to "https://oauth2.googleapis.com" to obtain a token - http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_response", __file__)), 200) - ) + http_mocker.post(AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_response", __file__)), 200)) @staticmethod - def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=None, meta_response_code: Optional[int]=200): - """" + def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str] = None, meta_response_code: Optional[int] = 200): + """ " Mock request to https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=false&alt=json in order to obtain sheets (streams) from the spreed_sheet_id provided. e.g. from response file @@ -79,12 +70,19 @@ def get_streams(http_mocker: HttpMocker, streams_response_file: Optional[str]=No if streams_response_file: http_mocker.get( RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(False).with_alt("json").build(), - HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code) + HttpResponse(json.dumps(find_template(streams_response_file, __file__)), meta_response_code), ) @staticmethod - def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_response_code: int=200, stream_name: Optional[str]=_STREAM_NAME, data_initial_range_response_file: Optional[str]=None, data_initial_response_code: Optional[int]=200): - """" + def get_schema( + http_mocker: HttpMocker, + headers_response_file: str, + headers_response_code: int = 200, + stream_name: Optional[str] = _STREAM_NAME, + data_initial_range_response_file: Optional[str] = None, + data_initial_response_code: Optional[int] = 200, + ): + """ " Mock request to 'https://sheets.googleapis.com/v4/spreadsheets/?includeGridData=true&ranges=!1:1&alt=json' to obtain headers data (keys) used for stream schema from the spreadsheet + sheet provided. For this we use range of first row in query. @@ -127,13 +125,20 @@ def get_schema(http_mocker: HttpMocker, headers_response_file: str, headers_resp ]}],}]}] """ http_mocker.get( - RequestBuilder().with_spreadsheet_id(_SPREADSHEET_ID).with_include_grid_data(True).with_ranges(f"{stream_name}!1:1").with_alt("json").build(), - HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code) + RequestBuilder() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_include_grid_data(True) + .with_ranges(f"{stream_name}!1:1") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(headers_response_file, __file__)), headers_response_code), ) @staticmethod - def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int=200, stream_name:Optional[str]=_STREAM_NAME): - """" + def get_stream_data( + http_mocker: HttpMocker, range_data_response_file: str, range_response_code: int = 200, stream_name: Optional[str] = _STREAM_NAME + ): + """ " Mock requests to 'https://sheets.googleapis.com/v4/spreadsheets//values:batchGet?ranges=!2:202&majorDimension=ROWS&alt=json' to obtain value ranges (data) for stream from the spreadsheet + sheet provided. For this we use range [2:202(2 + range in config which default is 200)]. @@ -156,8 +161,13 @@ def get_stream_data(http_mocker: HttpMocker, range_data_response_file: str, rang """ batch_request_ranges = f"{stream_name}!2:202" http_mocker.get( - RequestBuilder.get_account_endpoint().with_spreadsheet_id(_SPREADSHEET_ID).with_ranges(batch_request_ranges).with_major_dimension("ROWS").with_alt("json").build(), - HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code) + RequestBuilder.get_account_endpoint() + .with_spreadsheet_id(_SPREADSHEET_ID) + .with_ranges(batch_request_ranges) + .with_major_dimension("ROWS") + .with_alt("json") + .build(), + HttpResponse(json.dumps(find_template(range_data_response_file, __file__)), range_response_code), ) @HttpMocker() @@ -186,15 +196,12 @@ def test_given_service_authentication_error_when_check_then_status_is_failed(sel @HttpMocker() def test_invalid_credentials_error_message_when_check(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." - ) + assert str(exc_info.value) == ("Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.") @HttpMocker() def test_invalid_link_error_message_when_check(self, http_mocker: HttpMocker) -> None: @@ -216,9 +223,7 @@ def test_check_access_expired(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "invalid_permissions", 403) with pytest.raises(AirbyteTracedException) as exc_info: self._source.check(Mock(), self._config) - assert str(exc_info.value) == ( - "Config error: " - ) + assert str(exc_info.value) == ("Config error: ") @HttpMocker() def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) -> None: @@ -227,7 +232,10 @@ def test_check_expected_to_read_data_from_1_sheet(self, http_mocker: HttpMocker) connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f'Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. ' + assert ( + connection_status.message + == f"Unable to read the schema of sheet a_stream_name. Error: Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " + ) @HttpMocker() def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: @@ -236,7 +244,10 @@ def test_check_duplicated_headers(self, http_mocker: HttpMocker) -> None: connection_status = self._source.check(Mock(), self._config) assert connection_status.status == Status.FAILED - assert connection_status.message == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + assert ( + connection_status.message + == f"The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:{_STREAM_NAME}, headers:['header1']]" + ) @HttpMocker() def test_given_grid_sheet_type_with_at_least_one_row_when_discover_then_return_stream(self, http_mocker: HttpMocker) -> None: @@ -252,9 +263,9 @@ def test_discover_return_expected_schema(self, http_mocker: HttpMocker) -> None: # When we move to manifest only is possible that DynamicSchemaLoader will identify fields like age as integers # and addresses "oneOf": [{"type": ["null", "string"]}, {"type": ["null", "integer"]}] as it has mixed data expected_schemas_properties = { - _STREAM_NAME: {'age': {'type': 'string'}, 'name': {'type': 'string'}}, - _B_STREAM_NAME: {'email': {'type': 'string'}, 'name': {'type': 'string'}}, - _C_STREAM_NAME: {'address': {'type': 'string'}} + _STREAM_NAME: {"age": {"type": "string"}, "name": {"type": "string"}}, + _B_STREAM_NAME: {"email": {"type": "string"}, "name": {"type": "string"}}, + _C_STREAM_NAME: {"address": {"type": "string"}}, } GoogleSheetSourceTest.get_streams(http_mocker, "multiple_streams_schemas_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, f"multiple_streams_schemas_{_STREAM_NAME}_range", 200) @@ -293,13 +304,12 @@ def test_discover_could_not_run_discover(self, http_mocker: HttpMocker) -> None: @HttpMocker() def test_discover_invalid_credentials_error_message(self, http_mocker: HttpMocker) -> None: http_mocker.post( - AuthBuilder.get_token_endpoint().build(), - HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) + AuthBuilder.get_token_endpoint().build(), HttpResponse(json.dumps(find_template("auth_invalid_client", __file__)), 200) ) with pytest.raises(Exception) as exc_info: self._source.discover(Mock(), self._config) - expected_message = 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + expected_message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." assert str(exc_info.value) == expected_message @HttpMocker() @@ -350,7 +360,15 @@ def test_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range") GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 2 @@ -367,20 +385,23 @@ def test_when_read_multiple_streams_return_records(self, http_mocker: HttpMocker GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_B_STREAM_NAME}_range_2", stream_name=_B_STREAM_NAME) GoogleSheetSourceTest.get_stream_data(http_mocker, f"multiple_streams_schemas_{_C_STREAM_NAME}_range_2", stream_name=_C_STREAM_NAME) - configured_catalog = (CatalogBuilder().with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME). - with_json_schema({"properties": {'age': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_B_STREAM_NAME). - with_json_schema({"properties": {'email': {'type': 'string'}, 'name': {'type': 'string'}} - }) - ).with_stream( - ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME). - with_json_schema({"properties": {'address': {'type': 'string'}} - }) + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"age": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_B_STREAM_NAME) + .with_json_schema({"properties": {"email": {"type": "string"}, "name": {"type": "string"}}}) + ) + .with_stream( + ConfiguredAirbyteStreamBuilder().with_name(_C_STREAM_NAME).with_json_schema({"properties": {"address": {"type": "string"}}}) + ) + .build() ) - .build()) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 9 @@ -403,7 +424,15 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_2", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "read_records_range_with_dimensions_2") - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) assert len(output.records) == 5 @@ -414,21 +443,26 @@ def test_when_read_then_status_and_state_messages_emitted(self, http_mocker: Htt assert output.trace_messages[1].trace.stream_status.status == AirbyteStreamStatus.RUNNING assert output.trace_messages[2].trace.stream_status.status == AirbyteStreamStatus.COMPLETE - @HttpMocker() def test_read_429_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "rate_limit_error", 429) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account." assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -437,13 +471,19 @@ def test_read_403_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "invalid_permissions", 403) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" - ) + expected_message = f"Stopped syncing process. The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {_SPREADSHEET_ID}. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -452,14 +492,20 @@ def test_read_500_error(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range", 200) GoogleSheetSourceTest.get_stream_data(http_mocker, "internal_server_error", 500) - configured_catalog =CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) with patch("time.sleep"), patch("backoff._sync._maybe_call", side_effect=lambda value: 1): output = read(self._source, self._config, configured_catalog) - expected_message = ( - "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" - ) + expected_message = "Stopped syncing process. There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support" assert output.errors[0].trace.error.message == expected_message @HttpMocker() @@ -467,12 +513,18 @@ def test_read_empty_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_empty", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message @HttpMocker() @@ -480,10 +532,16 @@ def test_read_expected_data_on_1_sheet(self, http_mocker: HttpMocker) -> None: GoogleSheetSourceTest.get_streams(http_mocker, "read_records_meta", 200) GoogleSheetSourceTest.get_schema(http_mocker, "read_records_range_with_unexpected_extra_sheet", 200) - configured_catalog = CatalogBuilder().with_stream(ConfiguredAirbyteStreamBuilder().with_name(_STREAM_NAME).with_json_schema({"properties": {"header_1": { "type": ["null", "string"] }, "header_2": { "type": ["null", "string"] }}})).build() + configured_catalog = ( + CatalogBuilder() + .with_stream( + ConfiguredAirbyteStreamBuilder() + .with_name(_STREAM_NAME) + .with_json_schema({"properties": {"header_1": {"type": ["null", "string"]}, "header_2": {"type": ["null", "string"]}}}) + ) + .build() + ) output = read(self._source, self._config, configured_catalog) - expected_message = ( - f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " - ) + expected_message = f"Unexpected return result: Sheet {_STREAM_NAME} was expected to contain data on exactly 1 sheet. " assert output.errors[0].trace.error.internal_message == expected_message diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index b939910d57f7c..cdcbf8cc73145 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -191,7 +191,7 @@ def test_discover_invalid_credentials_error_message(mocker, invalid_config): source = SourceGoogleSheets() with pytest.raises(AirbyteTracedException) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - assert e.value.args[0] == 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + assert e.value.args[0] == "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." def test_get_credentials(invalid_config): @@ -223,12 +223,14 @@ def test_read_429_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) - expected_message = ( - "Rate limit has been reached. Please try later or request a higher quota for your account." - ) + expected_message = "Rate limit has been reached. Please try later or request a higher quota for your account." assert e.value.args[0] == expected_message @@ -243,7 +245,11 @@ def test_read_403_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) assert ( @@ -265,7 +271,11 @@ def test_read_500_error(mocker, invalid_config, catalog, caplog): sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet1, sheet1_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet1, sheet1_schema), + ) + ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) expected_message = ( @@ -303,8 +313,13 @@ def test_read_empty_sheet(invalid_config, mocker, catalog, caplog): sheet1 = "soccer_team" sheet2 = "soccer_team2" sheets = [ - Sheet(properties=SheetProperties(title=t), data=[{"test1": "12", "test2": "123"},]) - for t in [sheet1] + Sheet( + properties=SheetProperties(title=t), + data=[ + {"test1": "12", "test2": "123"}, + ], + ) + for t in [sheet1] ] mocker.patch.object( GoogleSheetsClient, @@ -328,7 +343,11 @@ def test_when_read_then_status_messages_emitted(mocker, spreadsheet, spreadsheet mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed @@ -346,7 +365,11 @@ def test_when_read_then_state_message_emitted(mocker, spreadsheet, spreadsheet_v mocker.patch.object(GoogleSheetsClient, "get_values", return_value=spreadsheet_values(spreadsheet_id)) sheet_schema = {"properties": {"ID": {"type": "string"}}} - catalog = ConfiguredAirbyteCatalog(streams=catalog((sheet_name, sheet_schema),)) + catalog = ConfiguredAirbyteCatalog( + streams=catalog( + (sheet_name, sheet_schema), + ) + ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) # stream started, stream running, 1 record, stream state, stream completed diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py index 605c45c1ea2fb..1451d4ccd7535 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py @@ -18,4 +18,11 @@ def greenhouse_slicer(): @pytest.fixture def greenhouse_substream_slicer(): parent_stream = MagicMock(spec=Stream) - return GreenHouseSubstreamSlicer(cursor_field='cursor_field', stream_slice_field='slice_field', parent_stream=parent_stream, parent_key='parent_key', parameters={}, request_cursor_field=None) + return GreenHouseSubstreamSlicer( + cursor_field="cursor_field", + stream_slice_field="slice_field", + parent_stream=parent_stream, + parent_key="parent_key", + parameters={}, + request_cursor_field=None, + ) diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py index 48db265f477f0..ba918ebcb935b 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -53,17 +53,12 @@ def test_sub_slicer(last_record, expected, records): @pytest.mark.parametrize( "stream_state, cursor_field, expected_state", [ - ({'cursor_field_1': '2022-09-05T10:10:10.000Z'}, 'cursor_field_1', {'cursor_field_1': '2022-09-05T10:10:10.000Z'}), - ({'cursor_field_2': '2022-09-05T10:10:100000Z'}, 'cursor_field_3', {}), - ({'cursor_field_4': None}, 'cursor_field_4', {}), - ({'cursor_field_5': ''}, 'cursor_field_5', {}), + ({"cursor_field_1": "2022-09-05T10:10:10.000Z"}, "cursor_field_1", {"cursor_field_1": "2022-09-05T10:10:10.000Z"}), + ({"cursor_field_2": "2022-09-05T10:10:100000Z"}, "cursor_field_3", {}), + ({"cursor_field_4": None}, "cursor_field_4", {}), + ({"cursor_field_5": ""}, "cursor_field_5", {}), ], - ids=[ - "cursor_value_present", - "cursor_value_not_present", - "cursor_value_is_None", - "cursor_value_is_empty_string" - ] + ids=["cursor_value_present", "cursor_value_not_present", "cursor_value_is_None", "cursor_value_is_empty_string"], ) def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer = GreenHouseSlicer(cursor_field=cursor_field, parameters={}, request_cursor_field=None) @@ -71,36 +66,30 @@ def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): slicer.set_initial_state(stream_state) assert slicer.get_stream_state() == expected_state + @pytest.mark.parametrize( "stream_state, initial_state, expected_state", [ ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'}}, - { - 'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}, - 'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'} - } - ), - ( - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T09:00:00.000Z'}}, - {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}} + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}, "id2": {"cursor_field": "2023-01-02T11:00:00.000Z"}}, ), ( - {}, - {}, - {} + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T09:00:00.000Z"}}, + {"id1": {"cursor_field": "2023-01-01T10:00:00.000Z"}}, ), + ({}, {}, {}), ], ids=[ "stream_state and initial_state have different keys", "stream_state and initial_state have overlapping keys with different values", - "stream_state and initial_state are empty" - ] + "stream_state and initial_state are empty", + ], ) def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, initial_state, expected_state): - slicer = greenhouse_substream_slicer + slicer = greenhouse_substream_slicer # Set initial state slicer._state = initial_state slicer.set_initial_state(stream_state) @@ -110,27 +99,11 @@ def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, @pytest.mark.parametrize( "first_record, second_record, expected_result", [ - ( - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ( - {'cursor_field': '2023-02-01T00:00:00.000Z'}, - {'cursor_field': '2023-01-01T00:00:00.000Z'}, - True - ), - ( - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - {'cursor_field': ''}, - True - ), - ( - {'cursor_field': ''}, - {'cursor_field': '2023-01-02T00:00:00.000Z'}, - False - ), - ] + ({"cursor_field": "2023-01-01T00:00:00.000Z"}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ({"cursor_field": "2023-02-01T00:00:00.000Z"}, {"cursor_field": "2023-01-01T00:00:00.000Z"}, True), + ({"cursor_field": "2023-01-02T00:00:00.000Z"}, {"cursor_field": ""}, True), + ({"cursor_field": ""}, {"cursor_field": "2023-01-02T00:00:00.000Z"}, False), + ], ) def test_is_greater_than_or_equal(greenhouse_substream_slicer, first_record, second_record, expected_result): slicer = greenhouse_substream_slicer diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index deecb0b377d3a..1acf8d5d1a821 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -81,9 +81,16 @@ def some_credentials_fixture(): def fake_properties_list(): return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] + @pytest.fixture(name="migrated_properties_list") def migrated_properties_list(): - return ["hs_v2_date_entered_prospect", "hs_v2_date_exited_prospect", "hs_v2_cumulative_time_in_prsopect", "hs_v2_some_other_property_in_prospect"] + return [ + "hs_v2_date_entered_prospect", + "hs_v2_date_exited_prospect", + "hs_v2_cumulative_time_in_prsopect", + "hs_v2_some_other_property_in_prospect", + ] + @pytest.fixture(name="api") def api(some_credentials): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py index 048142759ca2a..1927b02a17054 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/config_builder.py @@ -5,9 +5,7 @@ class ConfigBuilder: def __init__(self): - self._config = { - "enable_experimental_streams": True - } + self._config = {"enable_experimental_streams": True} def with_start_date(self, start_date: str): self._config["start_date"] = start_date diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py index 17ba71bebf3c1..e9695115f5a3c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/api.py @@ -26,8 +26,7 @@ def with_refresh_token(self, refresh_token: str): def build(self) -> HttpRequest: client_id, client_secret, refresh_token = self._params["client_id"], self._params["client_secret"], self._params["refresh_token"] return HttpRequest( - url=self.URL, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + url=self.URL, body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" ) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index d0258a26500a4..614c51e8ece61 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -29,12 +29,7 @@ def with_query(self, qp): return self def build(self) -> HttpRequest: - return HttpRequest( - url=self.URL, - query_params=self._query_params, - headers=self.headers, - body=self._request_body - ) + return HttpRequest(url=self.URL, query_params=self._query_params, headers=self.headers, body=self._request_body) class CRMStreamRequestBuilder(AbstractRequestBuilder): @@ -78,14 +73,7 @@ def _archived(self): @property def _query_params(self): - return [ - self._archived, - self._associations, - self._limit, - self._after, - self._dt_range, - self._properties - ] + return [self._archived, self._associations, self._limit, self._after, self._dt_range, self._properties] def build(self): q = "&".join(filter(None, self._query_params)) @@ -96,14 +84,7 @@ def build(self): class IncrementalCRMStreamRequestBuilder(CRMStreamRequestBuilder): @property def _query_params(self): - return [ - self._limit, - self._after, - self._dt_range, - self._archived, - self._associations, - self._properties - ] + return [self._limit, self._after, self._dt_range, self._archived, self._associations, self._properties] class OwnersArchivedStreamRequestBuilder(AbstractRequestBuilder): @@ -122,11 +103,14 @@ def _archived(self): @property def _query_params(self): - return filter(None, [ - self._limit, - self._after, - self._archived, - ]) + return filter( + None, + [ + self._limit, + self._after, + self._archived, + ], + ) def with_page_token(self, next_page_token: Dict): self._after = "&".join([f"{str(key)}={str(val)}" for key, val in next_page_token.items()]) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py index 87ba98af2b5a8..fe2f8d34067d3 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/contact_response_builder.py @@ -13,7 +13,6 @@ _MERGE_AUDITS_FIELD = "merge-audits" - def _get_template() -> Dict[str, Any]: return find_template("all_contacts", __file__) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py index 595a02232d439..f5bb912822f72 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/helpers.py @@ -12,7 +12,7 @@ def __init__( self, template: List[Any], records_path: Optional[Union[FieldPath, NestedPath]] = None, - pagination_strategy: Optional[PaginationStrategy] = None + pagination_strategy: Optional[PaginationStrategy] = None, ): self._response = template self._records: List[RecordBuilder] = [] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index ded25153204f4..7f594d984fdf9 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -9,13 +9,4 @@ class HubspotPaginationStrategy(PaginationStrategy): NEXT_PAGE_TOKEN = {"after": "256"} def update(self, response: Dict[str, Any]) -> None: - response["paging"] = { - "next": { - "link": "link_to_the_next_page", - **self.NEXT_PAGE_TOKEN - }, - "prev": { - "before": None, - "link": None - } - } + response["paging"] = {"next": {"link": "link_to_the_next_page", **self.NEXT_PAGE_TOKEN}, "prev": {"before": None, "link": None}} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index f5d4795c37752..6b02d952e14c0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -13,7 +13,7 @@ class HubspotStreamResponseBuilder(HttpResponseBuilder): @property def pagination_strategy(self): return self._pagination_strategy - + @classmethod def for_stream(cls, stream: str): return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py index 7bcddfe5e179d..5d47ea515d067 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py @@ -32,43 +32,60 @@ def tearDown(self) -> None: def test_read_multiple_contact_pages(self) -> None: first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( - cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), - stream=self.STREAM_NAME, - sync_mode=SyncMode.full_refresh + cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh ) self._http_mocker.assert_number_of_calls(first_page_request, 2) @@ -87,56 +104,73 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check first_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build() - second_page_request = ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + second_page_request = ( + ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset(str(_VID_OFFSET)).build() + ) self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ContactBuilder().with_form_submissions([ - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ContactsFormSubmissionsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ContactBuilder().with_form_submissions( + [ + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ContactsFormSubmissionsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py index 8fb6e598a395c..c4fd59c8cb88d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py @@ -37,25 +37,40 @@ def test_given_pagination_when_read_then_extract_records_from_both_pages(self) - self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).with_vid_offset(str(_VID_OFFSET)).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ]), - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder(), - ContactsListMembershipBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ] + ), + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder(), + ContactsListMembershipBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -67,14 +82,22 @@ def test_given_timestamp_before_start_date_when_read_then_filter_out(self) -> No self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(start_date + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(start_date - timedelta(days=10)), + ] + ), + ] + ) + .build(), + ) + output = self.read_from_stream( + self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh ) - output = self.read_from_stream(self.oauth_config(start_date=start_date.isoformat().replace("+00:00", "Z")), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -83,12 +106,18 @@ def test_given_state_when_read_then_filter_out(self) -> None: self.mock_response( self._http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_list_memberships([ - ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), - ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_list_memberships( + [ + ContactsListMembershipBuilder().with_timestamp(state_value + timedelta(days=10)), + ContactsListMembershipBuilder().with_timestamp(state_value - timedelta(days=10)), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py index 18c67317aebfb..a3d53d39f6afa 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py @@ -36,32 +36,49 @@ def test_read_multiple_contact_pages(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream(self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), self.STREAM_NAME, SyncMode.full_refresh) @@ -82,9 +99,8 @@ def test_read_from_incoming_state(self) -> None: AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), - stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) - ) + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ), ) ] @@ -94,39 +110,56 @@ def test_read_from_incoming_state(self) -> None: self.mock_response( self._http_mocker, first_page_request, - AllContactsResponseBuilder().with_pagination(vid_offset=_VID_OFFSET).with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_pagination(vid_offset=_VID_OFFSET) + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) self.mock_response( self._http_mocker, second_page_request, - AllContactsResponseBuilder().with_contacts([ - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ]), - ContactBuilder().with_merge_audits([ - ContactsMergeAuditsBuilder(), - ContactsMergeAuditsBuilder(), - ]), - ]).build(), + AllContactsResponseBuilder() + .with_contacts( + [ + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ] + ), + ContactBuilder().with_merge_audits( + [ + ContactsMergeAuditsBuilder(), + ContactsMergeAuditsBuilder(), + ] + ), + ] + ) + .build(), ) output = self.read_from_stream( cfg=self.oauth_config(start_date=_START_TIME_BEFORE_ANY_RECORD), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, - state=state + state=state, ) # We call the first page during check availability. And the sync actually starts with a request to the second page diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index d37005531ecf9..90aa742796b7d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -27,22 +27,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +81,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +98,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,8 +135,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 - diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py index d07dbd20985a9..1838e96f5c9d6 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_leads.py @@ -27,22 +27,21 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def request(self, page_token: Optional[Dict[str, str]] = None): - request_builder = CRMStreamRequestBuilder().for_entity( - self.OBJECT_TYPE - ).with_associations( - self.ASSOCIATIONS - ).with_properties( - list(self.PROPERTIES.keys()) + request_builder = ( + CRMStreamRequestBuilder() + .for_entity(self.OBJECT_TYPE) + .with_associations(self.ASSOCIATIONS) + .with_properties(list(self.PROPERTIES.keys())) ) if page_token: request_builder = request_builder.with_page_token(page_token) return request_builder.build() def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -82,11 +81,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records(self, def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response(with_pagination=True)) - self.mock_response( - http_mocker, - self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), - self.response() - ) + self.mock_response(http_mocker, self.request(page_token=self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN), self.response()) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 @@ -103,14 +98,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message(self, h @HttpMocker() def test_given_500_then_200_when_read_then_return_records(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) - self.mock_response( - http_mocker, - self.request(), - [ - HttpResponse(status_code=500, body="{}"), - self.response() - ] - ) + self.mock_response(http_mocker, self.request(), [HttpResponse(status_code=500, body="{}"), self.response()]) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 1 @@ -147,7 +135,5 @@ def test_given_one_page_when_read_then_get_records_with_flattened_properties(sel def test_given_incremental_sync_when_read_then_state_message_produced_and_state_match_latest_record(self, http_mocker: HttpMocker): self._set_up_requests(http_mocker) self.mock_response(http_mocker, self.request(), self.response()) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental - ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.incremental) assert len(output.state_messages) == 1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index d54a94ca6a9d8..6084c19532efc 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -16,6 +16,7 @@ class TestOwnersArchivedStream(HubspotTestCase): The test case contains a single test - this is just a sanity check, as the tested stream is identical to the `Owners` stream (which is covered by acceptance tests), except for a single url param. """ + SCOPES = ["crm.objects.owners.read"] CURSOR_FIELD = "updatedAt" STREAM_NAME = "owners_archived" @@ -28,10 +29,10 @@ def response_builder(self): return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) def response(self, with_pagination: bool = False): - record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( - FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at()) - ).with_field( - FieldPath("id"), self.OBJECT_ID + record = ( + self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)) + .with_field(FieldPath(self.CURSOR_FIELD), self.dt_str(self.updated_at())) + .with_field(FieldPath("id"), self.OBJECT_ID) ) response = self.response_builder.with_record(record) if with_pagination: @@ -54,7 +55,7 @@ def test_given_two_pages_when_read_stream_private_token_then_return_records(self self.mock_response( http_mocker, self.request().with_page_token(self.response_builder.pagination_strategy.NEXT_PAGE_TOKEN).build(), - self.response().build() + self.response().build(), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), self.STREAM_NAME, SyncMode.full_refresh) assert len(output.records) == 2 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 7e3889a69ee39..506f086366d22 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -51,17 +51,11 @@ def web_analytics_request( object_type: str, start_date: Optional[str] = None, end_date: Optional[str] = None, - first_page: bool = True + first_page: bool = True, ): start_date = start_date or cls.dt_str(cls.start_date()) end_date = end_date or cls.dt_str(cls.now()) - query = { - "limit": 100, - "occurredAfter": start_date, - "occurredBefore": end_date, - "objectId": object_id, - "objectType": object_type - } + query = {"limit": 100, "occurredAfter": start_date, "occurredBefore": end_date, "objectId": object_id, "objectType": object_type} if not first_page: query.update(cls.response_builder(stream).pagination_strategy.NEXT_PAGE_TOKEN) @@ -95,10 +89,10 @@ def mock_parent_object( ): response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: @@ -136,7 +130,7 @@ def test_given_one_page_when_read_stream_oauth_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.oauth_config(), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -154,7 +148,7 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -172,12 +166,12 @@ def test_given_two_pages_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, with_pagination=True) + self.web_analytics_response(stream_name, with_pagination=True), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type, first_page=False), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -196,7 +190,7 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, with_pagination=True, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_parent_object( http_mocker, @@ -205,17 +199,17 @@ def test_given_two_parent_pages_when_read_then_return_records( parent_stream_name, parent_stream_associations, first_page=False, - properties=list(self.PROPERTIES.keys()) + properties=list(self.PROPERTIES.keys()), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, "another_object_id", object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 2 @@ -236,7 +230,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return parent_stream_name, parent_stream_associations, list(self.PROPERTIES.keys()), - date_range=start_to_end + date_range=start_to_end, ) for dt_range in date_ranges: for _id in (self.OBJECT_ID, "another_object_id"): @@ -245,7 +239,7 @@ def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, _id, object_type, start, end), - web_analytics_response + web_analytics_response, ) config_start_dt = date_ranges[0][0] output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN, config_start_dt), stream_name, SyncMode.full_refresh) @@ -264,7 +258,7 @@ def test_given_error_response_when_read_analytics_then_get_trace_message( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=500, body="{}") + HttpResponse(status_code=500, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -285,10 +279,7 @@ def test_given_500_then_200_when_read_then_return_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - [ - HttpResponse(status_code=500, body="{}"), - self.web_analytics_response(stream_name) - ] + [HttpResponse(status_code=500, body="{}"), self.web_analytics_response(stream_name)], ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -299,12 +290,7 @@ def test_given_500_then_200_when_read_then_return_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_missing_scopes_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_oauth(http_mocker, self.ACCESS_TOKEN) self.mock_scopes(http_mocker, self.ACCESS_TOKEN, []) @@ -313,12 +299,7 @@ def test_given_missing_scopes_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_unauthorized_error_when_read_then_hault( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -328,7 +309,7 @@ def test_given_unauthorized_error_when_read_then_hault( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}") + HttpResponse(status_code=http.HTTPStatus.UNAUTHORIZED, body="{}"), ) with mock.patch("time.sleep"): output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) @@ -339,12 +320,7 @@ def test_given_unauthorized_error_when_read_then_hault( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_transformed_records( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): self.mock_custom_objects(http_mocker) self.mock_properties(http_mocker, object_type, self.PROPERTIES) @@ -354,7 +330,7 @@ def test_given_one_page_when_read_then_get_transformed_records( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name) + self.web_analytics_response(stream_name), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) record = output.records[0].record.data @@ -365,12 +341,7 @@ def test_given_one_page_when_read_then_get_transformed_records( @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_STREAMS) @HttpMocker() def test_given_one_page_when_read_then_get_no_records_filtered( - self, - stream_name, - parent_stream_name, - object_type, - parent_stream_associations, - http_mocker: HttpMocker + self, stream_name, parent_stream_name, object_type, parent_stream_associations, http_mocker: HttpMocker ): # validate that no filter is applied on the record set received from the API response self.mock_custom_objects(http_mocker) @@ -381,7 +352,7 @@ def test_given_one_page_when_read_then_get_no_records_filtered( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))) + self.web_analytics_response(stream_name, updated_on=self.dt_str(self.now() - timedelta(days=365))), ) output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.full_refresh) assert len(output.records) == 1 @@ -399,11 +370,9 @@ def test_given_incremental_sync_when_read_then_state_message_produced_and_state_ self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) - ) - output = self.read_from_stream( - self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) + output = self.read_from_stream(self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental) assert len(output.state_messages) == 1 cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(self.OBJECT_ID, {}).get(self.CURSOR_FIELD) @@ -423,15 +392,15 @@ def test_given_state_with_no_current_slice_when_read_then_current_slice_in_state self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) another_object_id = "another_object_id" current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}) - ) + stream_state=AirbyteStateBlob(**{another_object_id: {self.CURSOR_FIELD: self.dt_str(self.now())}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -453,14 +422,14 @@ def test_given_state_with_current_slice_when_read_then_state_is_updated( self.mock_response( http_mocker, self.web_analytics_request(stream_name, self.ACCESS_TOKEN, self.OBJECT_ID, object_type), - self.web_analytics_response(stream_name, id=self.OBJECT_ID) + self.web_analytics_response(stream_name, id=self.OBJECT_ID), ) current_state = AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name=stream_name), - stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}) - ) + stream_state=AirbyteStateBlob(**{self.OBJECT_ID: {self.CURSOR_FIELD: self.dt_str(self.start_date() - timedelta(days=30))}}), + ), ) output = self.read_from_stream( self.private_token_config(self.ACCESS_TOKEN), stream_name, SyncMode.incremental, state=[current_state] @@ -493,24 +462,23 @@ def mock_parent_object( date_range = date_range or (cls.dt_str(cls.start_date()), cls.dt_str(cls.now())) response_builder = cls.response_builder(stream_name) for object_id in object_ids: - record = cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)).with_field( - FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at()) - ).with_field( - FieldPath("id"), object_id + record = ( + cls.record_builder(stream_name, FieldPath(cls.PARENT_CURSOR_FIELD)) + .with_field(FieldPath(cls.PARENT_CURSOR_FIELD), cls.dt_str(cls.updated_at())) + .with_field(FieldPath("id"), object_id) ) response_builder = response_builder.with_record(record) if with_pagination: response_builder = response_builder.with_pagination() start, end = date_range - request_builder = IncrementalCRMStreamRequestBuilder().for_entity( - object_type - ).with_associations( - associations - ).with_dt_range( - ("startTimestamp", cls.dt_conversion(start)), - ("endTimestamp", cls.dt_conversion(end)) - ).with_properties(properties) + request_builder = ( + IncrementalCRMStreamRequestBuilder() + .for_entity(object_type) + .with_associations(associations) + .with_dt_range(("startTimestamp", cls.dt_conversion(start)), ("endTimestamp", cls.dt_conversion(end))) + .with_properties(properties) + ) if not first_page: request_builder = request_builder.with_page_token(response_builder.pagination_strategy.NEXT_PAGE_TOKEN) @@ -533,12 +501,8 @@ def test_given_one_page_when_read_stream_private_token_then_return_records( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_two_pages_when_read_then_return_records( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_two_pages_when_read_then_return_records( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_two_pages_when_read_then_return_records(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_two_pages_when_read_then_return_records(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_wide_date_range_and_multiple_parent_records_when_read_then_return_records( @@ -573,12 +537,8 @@ def test_given_missing_scopes_error_when_read_then_hault( ) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) - def test_given_unauthorized_error_when_read_then_hault( - self, stream_name, parent_stream_name, object_type, parent_stream_associations - ): - super().test_given_unauthorized_error_when_read_then_hault( - stream_name, parent_stream_name, object_type, parent_stream_associations - ) + def test_given_unauthorized_error_when_read_then_hault(self, stream_name, parent_stream_name, object_type, parent_stream_associations): + super().test_given_unauthorized_error_when_read_then_hault(stream_name, parent_stream_name, object_type, parent_stream_associations) @pytest.mark.parametrize(("stream_name", "parent_stream_name", "object_type", "parent_stream_associations"), CRM_INCREMENTAL_STREAMS) def test_given_one_page_when_read_then_get_transformed_records( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py index b57f773270f99..e83609da66b8c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_components.py @@ -25,32 +25,20 @@ "hs_v2_date_exited_prospect": {"type": ["null", "string"]}, "hs_date_exited_prospect": {"type": ["null", "string"]}, "hs_v2_some_other_field": {"type": ["null", "string"]}, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "hs_v2_date_entered_poetry": 1827 - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827}, { "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } + }, ), ( - { - "name": "Edgar Allen Poe", - "age": 215, - "birthplace": "Boston", - "properties": { - "hs_v2_date_entered_poetry": 1827 - } - }, + {"name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", "properties": {"hs_v2_date_entered_poetry": 1827}}, { "name": "Edgar Allen Poe", "age": 215, @@ -58,8 +46,8 @@ "properties": { "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 1827, - } - } + }, + }, ), ( { @@ -71,19 +59,15 @@ "name": "Edgar Allen Poe", "age": 215, "birthplace": "Boston", - } + }, ), ( - { - "name": "Edgar Allen Poe", - "hs_v2_date_entered_poetry": 1827, - "hs_date_entered_poetry": 9999 - }, + {"name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999}, { "name": "Edgar Allen Poe", "hs_v2_date_entered_poetry": 1827, "hs_date_entered_poetry": 9999, - } + }, ), ], ids=[ @@ -91,7 +75,7 @@ "Transforms record w/ flat properties", "Transform record w/ nested properties", "Does not transform record w/o need to transformation", - "Does not overwrite value for legacy field if legacy field exists" + "Does not overwrite value for legacy field if legacy field exists", ], ) def test_new_to_legacy_field_transformation(input, expected): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index b3c7e71e0d7fc..69577e3dcbbc8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -141,7 +141,7 @@ def test_cast_datetime(common_params, caplog): # if you find some diff locally try using "Ex: argument of type 'DateTime' is not iterable in the message". There could be a # difference in local environment when pendulum.parsing.__init__.py importing parse_iso8601. Anyway below is working fine # in container for now and I am not sure if this diff was just a problem with my setup. - "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object" + "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: expected string or bytes-like object", }, } assert expected_warning_message["log"]["message"] in caplog.text @@ -488,6 +488,7 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req assert len(records) == 11000 assert test_stream.state["updatedAt"] == test_stream._init_sync.to_iso8601_string() + def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common_params, fake_properties_list): """ If there are more than 10,000 records that would be returned by the Hubspot search endpoint, @@ -500,7 +501,7 @@ def test_search_based_incremental_stream_should_sort_by_id(requests_mock, common test_stream.associations = [] def random_date(start, end): - return pendulum.from_timestamp(random.randint(start, end)/1000).to_iso8601_string() + return pendulum.from_timestamp(random.randint(start, end) / 1000).to_iso8601_string() after = 0 @@ -518,17 +519,13 @@ def custom_callback(request, context): id = int(filters[2].get("value", 0)) next = int(after) + 100 results = [ - { - "id": f"{y + id}", - "updatedAt": random_date(min_time, max_time), - "after": after - } for y in range(int(after) + 1, next + 1) + {"id": f"{y + id}", "updatedAt": random_date(min_time, max_time), "after": after} for y in range(int(after) + 1, next + 1) ] context.status_code = 200 - if ((id + next) < 11000): + if (id + next) < 11000: return {"results": results, "paging": {"next": {"after": f"{next}"}}} else: - return {"results": results, "paging": {}} # Last page + return {"results": results, "paging": {}} # Last page properties_response = [ { @@ -787,6 +784,7 @@ def test_get_granted_scopes(requests_mock, mocker): assert expected_scopes == actual_scopes + def test_get_granted_scopes_retry(requests_mock, mocker): authenticator = mocker.Mock() expected_token = "the-token" diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 23ba97a303e70..6bfce6580a629 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -169,9 +169,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para contact_lists_v1_response = [ { "json": { - "contacts": [ - {"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]} - ] + "contacts": [{"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]}] }, "status_code": 200, } @@ -193,6 +191,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para records = read_full_refresh(stream) assert records + @pytest.mark.parametrize( "stream, endpoint, cursor_value", [ @@ -203,10 +202,12 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para ids=[ "Contacts stream with v2 field transformations", "Deals stream with v2 field transformations", - "DealsArchived stream with v2 field transformations" - ] + "DealsArchived stream with v2 field transformations", + ], ) -def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list): +def test_stream_read_with_legacy_field_transformation( + stream, endpoint, cursor_value, requests_mock, common_params, fake_properties_list, migrated_properties_list +): stream = stream(**common_params) responses = [ { @@ -221,7 +222,7 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", "hs_v2_cumulative_time_in_prsopect": "1 month", "hs_v2_some_other_property_in_prospect": "Great property", - } + }, } | cursor_value ], @@ -246,44 +247,40 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v records = read_full_refresh(stream) assert records expected_record = { - "id": "test_id", - "created": "2022-02-25T16:43:11Z", - "properties": { - "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "hs_v2_latest_time_in_prospect": "1 month", - "hs_v2_cumulative_time_in_prsopect": "1 month", - "hs_v2_some_other_property_in_prospect": "Great property", - "hs_time_in_prospect": "1 month", - "hs_date_exited_prospect": "2024-02-01T00:00:00Z", - }, - "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", - "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", - "properties_hs_v2_latest_time_in_prospect": "1 month", - "properties_hs_v2_cumulative_time_in_prsopect": "1 month", - "properties_hs_v2_some_other_property_in_prospect": "Great property", - "properties_hs_time_in_prospect": "1 month", - "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", - } | cursor_value + "id": "test_id", + "created": "2022-02-25T16:43:11Z", + "properties": { + "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "hs_v2_latest_time_in_prospect": "1 month", + "hs_v2_cumulative_time_in_prsopect": "1 month", + "hs_v2_some_other_property_in_prospect": "Great property", + "hs_time_in_prospect": "1 month", + "hs_date_exited_prospect": "2024-02-01T00:00:00Z", + }, + "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z", + "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z", + "properties_hs_v2_latest_time_in_prospect": "1 month", + "properties_hs_v2_cumulative_time_in_prsopect": "1 month", + "properties_hs_v2_some_other_property_in_prospect": "Great property", + "properties_hs_time_in_prospect": "1 month", + "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z", + } | cursor_value if isinstance(stream, Contacts): expected_record = expected_record | {"properties_hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} expected_record["properties"] = expected_record["properties"] | {"hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"} else: - expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z" } - expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z" } + expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z"} + expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z"} assert records[0] == expected_record - @pytest.mark.parametrize("sync_mode", [SyncMode.full_refresh, SyncMode.incremental]) -def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): +def test_crm_search_streams_with_no_associations(sync_mode, common_params, requests_mock, fake_properties_list): stream = DealSplits(**common_params) stream_state = { "type": "STREAM", - "stream": { - "stream_descriptor": { "name": "deal_splits" }, - "stream_state": { "updatedAt": "2021-01-01T00:00:00.000000Z" } - } + "stream": {"stream_descriptor": {"name": "deal_splits"}, "stream_state": {"updatedAt": "2021-01-01T00:00:00.000000Z"}}, } cursor_value = {"updatedAt": "2022-02-25T16:43:11Z"} responses = [ @@ -583,7 +580,7 @@ def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(req def test_get_custom_objects_metadata_success(requests_mock, custom_object_schema, expected_custom_object_json_schema, api): requests_mock.register_uri("GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) - for (entity, fully_qualified_name, schema, custom_properties) in api.get_custom_objects_metadata(): + for entity, fully_qualified_name, schema, custom_properties in api.get_custom_objects_metadata(): assert entity == "animals" assert fully_qualified_name == "p19936848_Animal" assert schema == expected_custom_object_json_schema diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py index 6687d9e16f0c4..eff56519383d9 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/test_streams.py @@ -35,12 +35,16 @@ def test_incremental_streams(self, configured_catalog, config, state): assert states, "insights should produce states" for state_msg in states: - stream_name, stream_state, state_keys_count = (state_msg.state.stream.stream_descriptor.name, - state_msg.state.stream.stream_state, - len(state_msg.state.stream.stream_state.dict())) + stream_name, stream_state, state_keys_count = ( + state_msg.state.stream.stream_descriptor.name, + state_msg.state.stream.stream_state, + len(state_msg.state.stream.stream_state.dict()), + ) assert stream_name == "user_insights", f"each state message should reference 'user_insights' stream, got {stream_name} instead" - assert isinstance(stream_state, AirbyteStateBlob), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" + assert isinstance( + stream_state, AirbyteStateBlob + ), f"Stream state should be type AirbyteStateBlob, got {type(stream_state)} instead" assert state_keys_count == 2, f"Stream state should contain 2 partition keys, got {state_keys_count} instead" @staticmethod diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index 92257eaab9e16..c68789661f2be 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -57,12 +57,11 @@ def fb_account_response_fixture(account_id, some_config, requests_mock): "access_token": "access_token", "category": "Software company", "id": f"act_{account_id}", - "paging": {"cursors": { - "before": "cursor", - "after": "cursor"}}, + "paging": {"cursors": {"before": "cursor", "after": "cursor"}}, "summary": {"total_count": 1}, - "status_code": 200 - }] + "status_code": 200, + } + ] } } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py index 7ac44159c282c..63fbf9f0c68b0 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/request_builder.py @@ -94,4 +94,3 @@ def build(self) -> HttpRequest: def _item_path(self) -> str: path_for_resource = "/" if self._item_id_is_sub_path else "" return f"{self._item_id}{path_for_resource}" if self._item_id else "" - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py index 4a1746e422aa0..67d14cad0a99e 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/response_builder.py @@ -23,20 +23,7 @@ def build_response( def get_account_response() -> HttpResponse: response = { - "data": [ - { - "id": PAGE_ID, - "name": "AccountName", - "instagram_business_account": { - "id": BUSINESS_ACCOUNT_ID - } - } - ], - "paging": { - "cursors": { - "before": "before_token", - "after": "after_token" - } - } - } + "data": [{"id": PAGE_ID, "name": "AccountName", "instagram_business_account": {"id": BUSINESS_ACCOUNT_ID}}], + "paging": {"cursors": {"before": "before_token", "after": "after_token"}}, + } return build_response(body=response, status_code=HTTPStatus.OK) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py index 850356c62f494..dc66be43595fa 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_api.py @@ -21,10 +21,7 @@ from .response_builder import get_account_response from .utils import config, read_output -_FIELDS = [ - "id", - "instagram_business_account" -] +_FIELDS = ["id", "instagram_business_account"] _STREAM_NAME = "Api" @@ -70,10 +67,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_accounts_with_no_instagram_business_account_field(self, http_mocker: HttpMocker) -> None: test = "not_instagram_business_account" mocked_response = json.dumps(find_template(f"api_for_{test}", __file__)) - http_mocker.get( - get_account_request().build(), - HttpResponse(mocked_response, 200) - ) + http_mocker.get(get_account_request().build(), HttpResponse(mocked_response, 200)) original_records = json.loads(mocked_response) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py index 429b75faa4254..c18e7d67030f7 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media.py @@ -29,7 +29,8 @@ "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -38,44 +39,21 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_CHILDREN_FIELDS = [ - "id", - "ig_id", - "media_type", - "media_url", - "owner", - "permalink", - "shortcode", - "thumbnail_url", - "timestamp", - "username" - ] - -_CHILDREN_IDS = [ - "07608776690540123", - "52896800415362123", - "39559889460059123", - "17359925580923123" - ] +_CHILDREN_FIELDS = ["id", "ig_id", "media_type", "media_url", "owner", "permalink", "shortcode", "thumbnail_url", "timestamp", "username"] + +_CHILDREN_IDS = ["07608776690540123", "52896800415362123", "39559889460059123", "17359925580923123"] _STREAM_NAME = "media" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(_FIELDS) -def _get_children_request(media_id:str) -> RequestBuilder: - return( - RequestBuilder.get_media_children_endpoint(item_id=media_id) - .with_fields(_CHILDREN_FIELDS) - ) +def _get_children_request(media_id: str) -> RequestBuilder: + return RequestBuilder.get_media_children_endpoint(item_id=media_id).with_fields(_CHILDREN_FIELDS) def _get_response() -> HttpResponseBuilder: @@ -93,6 +71,7 @@ def _record() -> RecordBuilder: record_id_path=FieldPath("id"), ) + class TestFullRefresh(TestCase): @staticmethod @@ -162,15 +141,12 @@ def test_given_one_page_has_children_field(self, http_mocker: HttpMocker) -> Non get_account_request().build(), get_account_response(), ) - http_mocker.get( - _get_request().build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) - ) + http_mocker.get(_get_request().build(), HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200)) for children_id in _CHILDREN_IDS: http_mocker.get( _get_children_request(children_id).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_children_for_{test}", __file__)), 200), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py index 11902699b40b7..2dde401ff564f 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_media_insights.py @@ -30,7 +30,8 @@ "id", "ig_id", "is_comment_enabled", - "like_count","media_type", + "like_count", + "media_type", "media_product_type", "media_url", "owner", @@ -39,9 +40,9 @@ "thumbnail_url", "timestamp", "username", - "children" + "children", ] -_PARENT_STREAM_NAME = 'media' +_PARENT_STREAM_NAME = "media" _STREAM_NAME = "media_insights" @@ -54,28 +55,38 @@ MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS = "35076616084176125" MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "35076616084176126" -REELS = 'reels' -VIDEO_FEED = 'video_feed' -VIDEO = 'video' -CAROUSEL_ALBUM = 'carousel_album' -GENERAL_MEDIA = 'general_media' -ERROR_POSTED_BEFORE_BUSINESS = 'error_posted_before_business' -ERROR_WITH_WRONG_PERMISSIONS = 'error_with_wrong_permissions' -ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = 'error_with_wrong_permissions_code_10' +REELS = "reels" +VIDEO_FEED = "video_feed" +VIDEO = "video" +CAROUSEL_ALBUM = "carousel_album" +GENERAL_MEDIA = "general_media" +ERROR_POSTED_BEFORE_BUSINESS = "error_posted_before_business" +ERROR_WITH_WRONG_PERMISSIONS = "error_with_wrong_permissions" +ERROR_WITH_WRONG_PERMISSIONS_CODE_10 = "error_with_wrong_permissions_code_10" _MEDIA_IDS = { REELS: MEDIA_ID_REELS, VIDEO_FEED: MEDIA_ID_VIDEO_FEED, VIDEO: MEDIA_ID_VIDEO, CAROUSEL_ALBUM: MEDIA_ID_CAROUSEL_ALBUM, - GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA + GENERAL_MEDIA: MEDIA_ID_GENERAL_MEDIA, } METRICS_GENERAL_MEDIA = ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"] _METRICS = { - MEDIA_ID_REELS: ["comments", "ig_reels_avg_watch_time", "ig_reels_video_view_total_time", "likes", "plays", "reach", "saved", "shares", - "ig_reels_aggregated_all_plays_count", "clips_replays_count"], + MEDIA_ID_REELS: [ + "comments", + "ig_reels_avg_watch_time", + "ig_reels_video_view_total_time", + "likes", + "plays", + "reach", + "saved", + "shares", + "ig_reels_aggregated_all_plays_count", + "clips_replays_count", + ], MEDIA_ID_VIDEO_FEED: ["impressions", "reach", "saved", "video_views"], MEDIA_ID_VIDEO: ["impressions", "reach", "saved", "video_views", "likes", "comments", "shares", "follows", "profile_visits"], MEDIA_ID_CAROUSEL_ALBUM: ["impressions", "reach", "saved", "video_views", "shares", "follows", "profile_visits"], @@ -83,29 +94,22 @@ # Reusing general media metrics for error scenarios MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS: METRICS_GENERAL_MEDIA, MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS: METRICS_GENERAL_MEDIA, - MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA + MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10: METRICS_GENERAL_MEDIA, } def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_media_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -114,15 +118,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -150,12 +152,14 @@ def test_instagram_insights_for_reels(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_REELS, metric=_METRICS[MEDIA_ID_REELS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -175,12 +179,14 @@ def test_instagram_insights_for_video_feed(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO_FEED, metric=_METRICS[MEDIA_ID_VIDEO_FEED]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -200,12 +206,14 @@ def test_instagram_insights_for_video(self, http_mocker: HttpMocker) -> None: ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_VIDEO, metric=_METRICS[MEDIA_ID_VIDEO]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -225,12 +233,14 @@ def test_instagram_insights_carousel_album(self, http_mocker: HttpMocker) -> Non ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_CAROUSEL_ALBUM, metric=_METRICS[MEDIA_ID_CAROUSEL_ALBUM]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -250,12 +260,14 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None ) http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -266,7 +278,6 @@ def test_instagram_insights_general_media(self, http_mocker: HttpMocker) -> None for metric in _METRICS[MEDIA_ID_GENERAL_MEDIA]: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_insights_error_posted_before_business(self, http_mocker: HttpMocker) -> None: test = ERROR_POSTED_BEFORE_BUSINESS @@ -275,18 +286,19 @@ def test_instagram_insights_error_posted_before_business(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS, metric=_METRICS[MEDIA_ID_ERROR_POSTED_BEFORE_BUSINESS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -305,18 +317,19 @@ def test_instagram_insights_error_with_wrong_permissions(self, http_mocker: Http get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) @@ -336,18 +349,19 @@ def test_instagram_insights_error_with_wrong_permissions_code_10(self, http_mock get_account_response(), ) http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) http_mocker.get( _get_child_request(media_id=MEDIA_ID_GENERAL_MEDIA, metric=_METRICS[MEDIA_ID_GENERAL_MEDIA]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{GENERAL_MEDIA}", __file__)), 200), ) http_mocker.get( - _get_child_request(media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10]).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + _get_child_request( + media_id=MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10, metric=_METRICS[MEDIA_ID_ERROR_WITH_WRONG_PERMISSIONS_CODE_10] + ).build(), + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py index abc137b78c719..7b645e6dfebfe 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_stories.py @@ -35,18 +35,14 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] _STREAM_NAME = "stories" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(FIELDS) def _get_response() -> HttpResponseBuilder: diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py index 1e9e3f0f47aa9..150d73703f75e 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_story_insights.py @@ -37,40 +37,32 @@ "shortcode", "thumbnail_url", "timestamp", - "username" + "username", ] -_PARENT_STREAM_NAME = 'stories' +_PARENT_STREAM_NAME = "stories" _STREAM_NAME = "story_insights" STORIES_ID = "3874523487643" STORIES_ID_ERROR_CODE_10 = "3874523487644" -HAPPY_PATH = 'story_insights_happy_path' -ERROR_10 = 'story_insights_error_code_10' +HAPPY_PATH = "story_insights_happy_path" +ERROR_10 = "story_insights_error_code_10" _METRICS = ["impressions", "reach", "replies", "follows", "profile_visits", "shares", "total_interactions"] - def _get_parent_request() -> RequestBuilder: - return ( - RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_limit(100) - .with_fields(PARENT_FIELDS) - ) + return RequestBuilder.get_stories_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_limit(100).with_fields(PARENT_FIELDS) def _get_child_request(media_id, metric) -> RequestBuilder: - return ( - RequestBuilder.get_media_insights_endpoint(item_id=media_id) - .with_custom_param("metric", metric, with_format=True) - ) + return RequestBuilder.get_media_insights_endpoint(item_id=media_id).with_custom_param("metric", metric, with_format=True) def _get_response(stream_name: str, test: str = None, with_pagination_strategy: bool = True) -> HttpResponseBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" kwargs = { "response_template": find_template(f"{stream_name}{scenario}", __file__), "records_path": FieldPath("data"), @@ -79,15 +71,13 @@ def _get_response(stream_name: str, test: str = None, with_pagination_strategy: if with_pagination_strategy: kwargs["pagination_strategy"] = InstagramPaginationStrategy(request=_get_parent_request().build(), next_page_token=NEXT_PAGE_TOKEN) - return create_response_builder( - **kwargs - ) + return create_response_builder(**kwargs) def _record(stream_name: str, test: str = None) -> RecordBuilder: - scenario = '' + scenario = "" if test: - scenario = f'_for_{test}' + scenario = f"_for_{test}" return create_record_builder( response_template=find_template(f"{stream_name}{scenario}", __file__), records_path=FieldPath("data"), @@ -117,12 +107,14 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: # Mocking parent stream http_mocker.get( _get_parent_request().build(), - _get_response(stream_name=_PARENT_STREAM_NAME, test=test).with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)).build(), + _get_response(stream_name=_PARENT_STREAM_NAME, test=test) + .with_record(_record(stream_name=_PARENT_STREAM_NAME, test=test)) + .build(), ) http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 200), ) output = self._read(config_=config()) @@ -133,7 +125,6 @@ def test_instagram_story_insights(self, http_mocker: HttpMocker) -> None: for metric in _METRICS: assert metric in output.records[0].record.data - @HttpMocker() def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocker) -> None: test = ERROR_10 @@ -143,18 +134,17 @@ def test_instagram_story_insights_for_error_code_30(self, http_mocker: HttpMocke ) # Mocking parent stream http_mocker.get( - _get_parent_request().build(), - HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) + _get_parent_request().build(), HttpResponse(json.dumps(find_template(f"{_PARENT_STREAM_NAME}_for_{test}", __file__)), 200) ) # Good response http_mocker.get( _get_child_request(media_id=STORIES_ID, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{HAPPY_PATH}", __file__)), 200), ) # error 10 http_mocker.get( _get_child_request(media_id=STORIES_ID_ERROR_CODE_10, metric=_METRICS).build(), - HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400) + HttpResponse(json.dumps(find_template(f"{_STREAM_NAME}_for_{test}", __file__)), 400), ) output = self._read(config_=config()) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py index a9a5d717fc0f1..5fd309c01a2bd 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_user_lifetime_insights.py @@ -29,21 +29,24 @@ def _get_request() -> RequestBuilder: return ( RequestBuilder.get_user_lifetime_insights_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_custom_param("metric", "follower_demographics").with_custom_param("period", "lifetime").with_custom_param("metric_type", "total_value").with_limit(100) + .with_custom_param("metric", "follower_demographics") + .with_custom_param("period", "lifetime") + .with_custom_param("metric_type", "total_value") + .with_limit(100) ) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), record_cursor_path=FieldPath(_CURSOR_FIELD), ) @@ -76,4 +79,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) # each breakdown should produce a record assert len(output.records) == 3 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py index 2e50aae606f2d..cf92237786fc3 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/integration/test_users.py @@ -31,30 +31,27 @@ "name", "profile_picture_url", "username", - "website" + "website", ] _STREAM_NAME = "users" def _get_request() -> RequestBuilder: - return ( - RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID) - .with_fields(_FIELDS) - ) + return RequestBuilder.get_users_endpoint(item_id=BUSINESS_ACCOUNT_ID).with_fields(_FIELDS) def _get_response() -> HttpResponseBuilder: return create_response_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), ) def _record() -> RecordBuilder: return create_record_builder( response_template=find_template(_STREAM_NAME, __file__), - records_path=FieldPath('data'), + records_path=FieldPath("data"), record_id_path=FieldPath("id"), ) @@ -83,4 +80,3 @@ def test_read_records(self, http_mocker: HttpMocker) -> None: output = self._read(config_=config()) assert len(output.records) == 1 - diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py index ecf06f7b64e59..4ccb16aade932 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/records.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/records.py @@ -1,77 +1,53 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. children_record = { - "children": { - "data": [ - { - "id": "7608776690540" - }, - { - "id": "2896800415362" - }, - { - "id": "9559889460059" - }, - { - "id": "7359925580923" - } - ] - } + "children": {"data": [{"id": "7608776690540"}, {"id": "2896800415362"}, {"id": "9559889460059"}, {"id": "7359925580923"}]} } expected_children_transformed = { - "children": - [ - { + "children": [ + { "id": "7608776690540", "ig_id": "2521545917836833225", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=tTUSyCXbN40Q7kNvgF-k3H6&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYAe4PKmuen4Dryt4sMEbfvrW2_eANbY1AEdl7gHG0a3mw&oe=66701C8F", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "2896800415362", "ig_id": "2521545917736276706", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=100&ccb=1-7&_nc_sid=18de74&_nc_ohc=9qJ5-fOc9lcQ7kNvgFirW6U&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBSuGRqMEzjxHri30L5NDs0irt7_7h-arKTYS8inrL56g&oe=66702E9A", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "9559889460059", "ig_id": "2521545917845406325", "media_type": "IMAGE", "media_url": "https://fake_url?_nc_cat=110&ccb=1-7&_nc_sid=18de74&_nc_ohc=QOtZzdxFjusQ7kNvgEqsuc2&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBPBGVS3NYW-h8oLwam_rWub-mE-9MLGc1EDVHtLJ2DBQ&oe=66702DE1", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - }, - { + "username": "username", + }, + { "id": "7359925580923", "ig_id": "2521545555591565193", "media_type": "VIDEO", "media_url": "https://fake_url?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNhcm91c2VsX2l0ZW0udW5rbm93bi1DMy40ODAuZGFzaF9iYXNlbGluZV8xX3YxIn0&_nc_ht=fakecontent&_nc_cat=108&vs=863753484982045_2117350142", - "owner": { - "id": "id" - }, + "owner": {"id": "id"}, "shortcode": "shortcode", "thumbnail_url": "https://fake_url?_nc_cat=108&ccb=1-7&_nc_sid=18de74&_nc_ohc=pJkRskDC80UQ7kNvgFn3i4H&_nc_ht=fakecontent&edm=AEQ6tj4EAAAA&oh=00_AYBKK27CU9dvjiqPi9a4JKUIICp26HZ074-vgz0OVKFkbw&oe=66702104", "timestamp": "2021-03-03T22:48:39+00:00", - "username": "username" - } - ] + "username": "username", + }, + ] } clear_url_record = { @@ -85,293 +61,66 @@ } breakdowns_record = { - "name": "follower_demographics", - "period": "lifetime", - "title": "Follower demographics", - "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", - "total_value": { - "breakdowns": [ + "name": "follower_demographics", + "period": "lifetime", + "title": "Follower demographics", + "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", + "total_value": { + "breakdowns": [ { - "dimension_keys": [ - "city" - ], - "results": [ - { - "dimension_values": [ - "London, England" - ], - "value": 263 - }, - { - "dimension_values": [ - "Sydney, New South Wales" - ], - "value": 467 - }, - { - "dimension_values": [ - "Algiers, Algiers Province" - ], - "value": 58 - }, - { - "dimension_values": [ - "Casablanca, Grand Casablanca" - ], - "value": 71 - }, - { - "dimension_values": [ - "São Paulo, São Paulo (state)" - ], - "value": 139 - }, - { - "dimension_values": [ - "Rio de Janeiro, Rio de Janeiro (state)" - ], - "value": 44 - }, - { - "dimension_values": [ - "Perth, Western Australia" - ], - "value": 180 - }, - { - "dimension_values": [ - "Berlin, Berlin" - ], - "value": 47 - }, - { - "dimension_values": [ - "Kolkata, West Bengal" - ], - "value": 85 - }, - { - "dimension_values": [ - "Phoenix, Arizona" - ], - "value": 39 - }, - { - "dimension_values": [ - "Lagos, Lagos State" - ], - "value": 40 - }, - { - "dimension_values": [ - "Dublin, Dublin" - ], - "value": 65 - }, - { - "dimension_values": [ - "Pune, Maharashtra" - ], - "value": 72 - }, - { - "dimension_values": [ - "Wollongong, New South Wales" - ], - "value": 43 - }, - { - "dimension_values": [ - "Christchurch, Canterbury" - ], - "value": 42 - }, - { - "dimension_values": [ - "Jakarta, Jakarta" - ], - "value": 46 - }, - { - "dimension_values": [ - "Pretoria, Gauteng" - ], - "value": 54 - }, - { - "dimension_values": [ - "Buenos Aires, Ciudad Autónoma de Buenos Aires" - ], - "value": 41 - }, - { - "dimension_values": [ - "Gold Coast, Queensland" - ], - "value": 98 - }, - { - "dimension_values": [ - "Sunshine Coast, Queensland" - ], - "value": 37 - }, - { - "dimension_values": [ - "Melbourne, Victoria" - ], - "value": 338 - }, - { - "dimension_values": [ - "Gurugram, Haryana" - ], - "value": 52 - }, - { - "dimension_values": [ - "Delhi, Delhi" - ], - "value": 194 - }, - { - "dimension_values": [ - "Los Angeles, California" - ], - "value": 66 - }, - { - "dimension_values": [ - "Madrid, Comunidad de Madrid" - ], - "value": 65 - }, - { - "dimension_values": [ - "Lahore, Punjab" - ], - "value": 41 - }, - { - "dimension_values": [ - "Brisbane, Queensland" - ], - "value": 160 - }, - { - "dimension_values": [ - "Adelaide, South Australia" - ], - "value": 93 - }, - { - "dimension_values": [ - "Canberra, Australian Capital Territory" - ], - "value": 45 - }, - { - "dimension_values": [ - "Lima, Lima Region" - ], - "value": 43 - }, - { - "dimension_values": [ - "Istanbul, Istanbul Province" - ], - "value": 57 - }, - { - "dimension_values": [ - "Toronto, Ontario" - ], - "value": 40 - }, - { - "dimension_values": [ - "Chennai, Tamil Nadu" - ], - "value": 82 - }, - { - "dimension_values": [ - "Mexico City, Distrito Federal" - ], - "value": 66 - }, - { - "dimension_values": [ - "Auckland, Auckland Region" - ], - "value": 98 - }, - { - "dimension_values": [ - "Cape Town, Western Cape" - ], - "value": 172 - }, - { - "dimension_values": [ - "New York, New York" - ], - "value": 139 - }, - { - "dimension_values": [ - "Cairo, Cairo Governorate" - ], - "value": 45 - }, - { - "dimension_values": [ - "Dubai, Dubai" - ], - "value": 57 - }, - { - "dimension_values": [ - "Santiago, Santiago Metropolitan Region" - ], - "value": 73 - }, - { - "dimension_values": [ - "Mumbai, Maharashtra" - ], - "value": 195 - }, - { - "dimension_values": [ - "Bangalore, Karnataka" - ], - "value": 195 - }, - { - "dimension_values": [ - "Nairobi, Nairobi" - ], - "value": 50 - }, - { - "dimension_values": [ - "Johannesburg, Gauteng" - ], - "value": 50 - }, - { - "dimension_values": [ - "Hyderabad, Telangana" - ], - "value": 49 - } - ] + "dimension_keys": ["city"], + "results": [ + {"dimension_values": ["London, England"], "value": 263}, + {"dimension_values": ["Sydney, New South Wales"], "value": 467}, + {"dimension_values": ["Algiers, Algiers Province"], "value": 58}, + {"dimension_values": ["Casablanca, Grand Casablanca"], "value": 71}, + {"dimension_values": ["São Paulo, São Paulo (state)"], "value": 139}, + {"dimension_values": ["Rio de Janeiro, Rio de Janeiro (state)"], "value": 44}, + {"dimension_values": ["Perth, Western Australia"], "value": 180}, + {"dimension_values": ["Berlin, Berlin"], "value": 47}, + {"dimension_values": ["Kolkata, West Bengal"], "value": 85}, + {"dimension_values": ["Phoenix, Arizona"], "value": 39}, + {"dimension_values": ["Lagos, Lagos State"], "value": 40}, + {"dimension_values": ["Dublin, Dublin"], "value": 65}, + {"dimension_values": ["Pune, Maharashtra"], "value": 72}, + {"dimension_values": ["Wollongong, New South Wales"], "value": 43}, + {"dimension_values": ["Christchurch, Canterbury"], "value": 42}, + {"dimension_values": ["Jakarta, Jakarta"], "value": 46}, + {"dimension_values": ["Pretoria, Gauteng"], "value": 54}, + {"dimension_values": ["Buenos Aires, Ciudad Autónoma de Buenos Aires"], "value": 41}, + {"dimension_values": ["Gold Coast, Queensland"], "value": 98}, + {"dimension_values": ["Sunshine Coast, Queensland"], "value": 37}, + {"dimension_values": ["Melbourne, Victoria"], "value": 338}, + {"dimension_values": ["Gurugram, Haryana"], "value": 52}, + {"dimension_values": ["Delhi, Delhi"], "value": 194}, + {"dimension_values": ["Los Angeles, California"], "value": 66}, + {"dimension_values": ["Madrid, Comunidad de Madrid"], "value": 65}, + {"dimension_values": ["Lahore, Punjab"], "value": 41}, + {"dimension_values": ["Brisbane, Queensland"], "value": 160}, + {"dimension_values": ["Adelaide, South Australia"], "value": 93}, + {"dimension_values": ["Canberra, Australian Capital Territory"], "value": 45}, + {"dimension_values": ["Lima, Lima Region"], "value": 43}, + {"dimension_values": ["Istanbul, Istanbul Province"], "value": 57}, + {"dimension_values": ["Toronto, Ontario"], "value": 40}, + {"dimension_values": ["Chennai, Tamil Nadu"], "value": 82}, + {"dimension_values": ["Mexico City, Distrito Federal"], "value": 66}, + {"dimension_values": ["Auckland, Auckland Region"], "value": 98}, + {"dimension_values": ["Cape Town, Western Cape"], "value": 172}, + {"dimension_values": ["New York, New York"], "value": 139}, + {"dimension_values": ["Cairo, Cairo Governorate"], "value": 45}, + {"dimension_values": ["Dubai, Dubai"], "value": 57}, + {"dimension_values": ["Santiago, Santiago Metropolitan Region"], "value": 73}, + {"dimension_values": ["Mumbai, Maharashtra"], "value": 195}, + {"dimension_values": ["Bangalore, Karnataka"], "value": 195}, + {"dimension_values": ["Nairobi, Nairobi"], "value": 50}, + {"dimension_values": ["Johannesburg, Gauteng"], "value": 50}, + {"dimension_values": ["Hyderabad, Telangana"], "value": 49}, + ], } - ] - }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + ] + }, + "id": "17841457631192237/insights/follower_demographics/lifetime", +} expected_breakdown_record_transformed = { "name": "follower_demographics", @@ -379,153 +128,121 @@ "title": "Follower demographics", "description": "The demographic characteristics of followers, including countries, cities and gender distribution.", "value": { - "London, England": 263, - "Sydney, New South Wales": 467, - "Algiers, Algiers Province": 58, - "Casablanca, Grand Casablanca": 71, - "São Paulo, São Paulo (state)": 139, - "Rio de Janeiro, Rio de Janeiro (state)": 44, - "Perth, Western Australia": 180, - "Berlin, Berlin": 47, - "Kolkata, West Bengal": 85, - "Phoenix, Arizona": 39, - "Lagos, Lagos State": 40, - "Dublin, Dublin": 65, - "Pune, Maharashtra": 72, - "Wollongong, New South Wales": 43, - "Christchurch, Canterbury": 42, - "Jakarta, Jakarta": 46, - "Pretoria, Gauteng": 54, - "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, - "Gold Coast, Queensland": 98, - "Sunshine Coast, Queensland": 37, - "Melbourne, Victoria": 338, - "Gurugram, Haryana": 52, - "Delhi, Delhi": 194, - "Los Angeles, California": 66, - "Madrid, Comunidad de Madrid": 65, - "Lahore, Punjab": 41, - "Brisbane, Queensland": 160, - "Adelaide, South Australia": 93, - "Canberra, Australian Capital Territory": 45, - "Lima, Lima Region": 43, - "Istanbul, Istanbul Province": 57, - "Toronto, Ontario": 40, - "Chennai, Tamil Nadu": 82, - "Mexico City, Distrito Federal": 66, - "Auckland, Auckland Region": 98, - "Cape Town, Western Cape": 172, - "New York, New York": 139, - "Cairo, Cairo Governorate": 45, - "Dubai, Dubai": 57, - "Santiago, Santiago Metropolitan Region": 73, - "Mumbai, Maharashtra": 195, - "Bangalore, Karnataka": 195, - "Nairobi, Nairobi": 50, - "Johannesburg, Gauteng": 50, - "Hyderabad, Telangana": 49 + "London, England": 263, + "Sydney, New South Wales": 467, + "Algiers, Algiers Province": 58, + "Casablanca, Grand Casablanca": 71, + "São Paulo, São Paulo (state)": 139, + "Rio de Janeiro, Rio de Janeiro (state)": 44, + "Perth, Western Australia": 180, + "Berlin, Berlin": 47, + "Kolkata, West Bengal": 85, + "Phoenix, Arizona": 39, + "Lagos, Lagos State": 40, + "Dublin, Dublin": 65, + "Pune, Maharashtra": 72, + "Wollongong, New South Wales": 43, + "Christchurch, Canterbury": 42, + "Jakarta, Jakarta": 46, + "Pretoria, Gauteng": 54, + "Buenos Aires, Ciudad Autónoma de Buenos Aires": 41, + "Gold Coast, Queensland": 98, + "Sunshine Coast, Queensland": 37, + "Melbourne, Victoria": 338, + "Gurugram, Haryana": 52, + "Delhi, Delhi": 194, + "Los Angeles, California": 66, + "Madrid, Comunidad de Madrid": 65, + "Lahore, Punjab": 41, + "Brisbane, Queensland": 160, + "Adelaide, South Australia": 93, + "Canberra, Australian Capital Territory": 45, + "Lima, Lima Region": 43, + "Istanbul, Istanbul Province": 57, + "Toronto, Ontario": 40, + "Chennai, Tamil Nadu": 82, + "Mexico City, Distrito Federal": 66, + "Auckland, Auckland Region": 98, + "Cape Town, Western Cape": 172, + "New York, New York": 139, + "Cairo, Cairo Governorate": 45, + "Dubai, Dubai": 57, + "Santiago, Santiago Metropolitan Region": 73, + "Mumbai, Maharashtra": 195, + "Bangalore, Karnataka": 195, + "Nairobi, Nairobi": 50, + "Johannesburg, Gauteng": 50, + "Hyderabad, Telangana": 49, }, - "id": "17841457631192237/insights/follower_demographics/lifetime" - } + "id": "17841457631192237/insights/follower_demographics/lifetime", +} insights_record = { "data": [ - { - "name": "comments", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title1", - "description": "Description1.", - "id": "insta_id/insights/comments/lifetime" - }, - { - "name": "ig_reels_avg_watch_time", - "period": "lifetime", - "values": [ - { - "value": 11900 - } - ], - "title": "2", - "description": "Description2.", - "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime" - }, - { - "name": "ig_reels_video_view_total_time", - "period": "lifetime", - "values": [ - { - "value": 25979677 - } - ], - "title": "title3", - "description": "Description3.", - "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime" - }, - { - "name": "likes", - "period": "lifetime", - "values": [ - { - "value": 102 - } - ], - "title": "title4", - "description": "Description4.", - "id": "insta_id/insights/likes/lifetime" - }, - { - "name": "plays", - "period": "lifetime", - "values": [ - { - "value": 2176 - } - ], - "title": "title5", - "description": "Description5.", - "id": "insta_id/insights/plays/lifetime" - }, - { - "name": "reach", - "period": "lifetime", - "values": [ - { - "value": 1842 - } - ], - "title": "title6", - "description": "Description6.", - "id": "insta_id/insights/reach/lifetime" - }, - { - "name": "saved", - "period": "lifetime", - "values": [ - { - "value": 7 - } - ], - "title": "title7", - "description": "Description7.", - "id": "insta_id/insights/saved/lifetime" - }, - { - "name": "shares", - "period": "lifetime", - "values": [ - { - "value": 1 - } - ], - "title": "title8", - "description": "Description8.", - "id": "insta_id/insights/shares/lifetime" - } + { + "name": "comments", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title1", + "description": "Description1.", + "id": "insta_id/insights/comments/lifetime", + }, + { + "name": "ig_reels_avg_watch_time", + "period": "lifetime", + "values": [{"value": 11900}], + "title": "2", + "description": "Description2.", + "id": "insta_id/insights/ig_reels_avg_watch_time/lifetime", + }, + { + "name": "ig_reels_video_view_total_time", + "period": "lifetime", + "values": [{"value": 25979677}], + "title": "title3", + "description": "Description3.", + "id": "insta_id/insights/ig_reels_video_view_total_time/lifetime", + }, + { + "name": "likes", + "period": "lifetime", + "values": [{"value": 102}], + "title": "title4", + "description": "Description4.", + "id": "insta_id/insights/likes/lifetime", + }, + { + "name": "plays", + "period": "lifetime", + "values": [{"value": 2176}], + "title": "title5", + "description": "Description5.", + "id": "insta_id/insights/plays/lifetime", + }, + { + "name": "reach", + "period": "lifetime", + "values": [{"value": 1842}], + "title": "title6", + "description": "Description6.", + "id": "insta_id/insights/reach/lifetime", + }, + { + "name": "saved", + "period": "lifetime", + "values": [{"value": 7}], + "title": "title7", + "description": "Description7.", + "id": "insta_id/insights/saved/lifetime", + }, + { + "name": "shares", + "period": "lifetime", + "values": [{"value": 1}], + "title": "title8", + "description": "Description8.", + "id": "insta_id/insights/shares/lifetime", + }, ] } diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 4f371e0fc9cf0..a811b568b0354 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -24,23 +24,11 @@ account_url_response = { - "data": [ - { - "id": "page_id", - "name": "Airbyte", - "instagram_business_account": { - "id": "instagram_business_account_id" - } - } - ], - "paging": { - "cursors": { - "before": "before", - "after": "after" - } - } + "data": [{"id": "page_id", "name": "Airbyte", "instagram_business_account": {"id": "instagram_business_account_id"}}], + "paging": {"cursors": {"before": "before", "after": "after"}}, } + def test_check_connection_ok(api, requests_mock, some_config): requests_mock.register_uri("GET", account_url, [{"json": account_url_response}]) ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 58fe97e5002fb..242ecab62d536 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -30,7 +30,6 @@ def test_state_is_not_outdated(api, config): assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) - def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" @@ -42,7 +41,6 @@ def test_user_insights_read(api, config, user_insight_data, requests_mock): assert records - @pytest.mark.parametrize( "values,slice_dates,expected", [ diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index 83fa25aa174b9..4af8ced9e23b4 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -54,6 +54,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourceIterable(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py index 2edcb5eb27c76..216336e503031 100644 --- a/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-jina-ai-reader/unit_tests/test_config_migrations.py @@ -16,7 +16,7 @@ def test_should_migrate(): def test__modify_and_save(): source = SourceJinaAiReader() user_config = {"search_prompt": "What is AI"} - expected = {"search_prompt": "What%20is%20AI" } + expected = {"search_prompt": "What%20is%20AI"} modified_config = JinaAiReaderConfigMigration.modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config=user_config) assert modified_config["search_prompt"] == expected["search_prompt"] assert modified_config.get("search_prompt") diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 1d9f622bd4a71..61920cadf24b1 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -173,7 +173,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class FullRefreshJiraStream(JiraStream): - """ This is a temporary solution to avoid incorrect state handling. See comments below for more info: diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py index c0306ea87a4a1..382e07b3df43a 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py @@ -40,6 +40,7 @@ def _response_template() -> Dict[str, Any]: with open(os.path.join(os.path.dirname(__file__), "..", "responses", "issues.json")) as response_file_handler: return json.load(response_file_handler) + def _create_response() -> HttpResponseBuilder: return create_response_builder( response_template=_response_template(), @@ -60,15 +61,19 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: config = _create_config().build() datetime_with_timezone = "2023-11-01T00:00:00.000-0800" timestamp_with_timezone = 1698825600000 - state = StateBuilder().with_stream_state( - "issues", - { - "use_global_cursor":False, - "state": {"updated": datetime_with_timezone}, - "lookback_window": 2, - "states": [{"partition":{"parent_slice":{},"project_id":"10025"},"cursor":{"updated": datetime_with_timezone}}] - } - ).build() + state = ( + StateBuilder() + .with_stream_state( + "issues", + { + "use_global_cursor": False, + "state": {"updated": datetime_with_timezone}, + "lookback_window": 2, + "states": [{"partition": {"parent_slice": {}, "project_id": "10025"}, "cursor": {"updated": datetime_with_timezone}}], + }, + ) + .build() + ) http_mocker.get( HttpRequest( f"https://{_DOMAIN}/rest/api/3/search", @@ -77,7 +82,7 @@ def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: "jql": f"updated >= {timestamp_with_timezone} ORDER BY updated asc", "expand": "renderedFields,transitions,changelog", "maxResults": "50", - } + }, ), _create_response().with_record(_create_record()).with_record(_create_record()).build(), ) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index b6d05c6f4fbc0..32c0888a608ee 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -50,9 +50,7 @@ def test_application_roles_stream_http_error(config, application_roles_response) responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/applicationrole", json={"error": "not found"}, status=404) stream = find_stream("application_roles", config) - with pytest.raises( - AirbyteTracedException, match="Not found. The requested resource was not found on the server" - ): + with pytest.raises(AirbyteTracedException, match="Not found. The requested resource was not found on the server"): list(read_full_refresh(stream)) @@ -82,10 +80,7 @@ def test_board_stream_forbidden(config, boards_response, caplog): ) stream = find_stream("boards", config) - with pytest.raises( - AirbyteTracedException, - match="Forbidden. You don't have permission to access this resource." - ): + with pytest.raises(AirbyteTracedException, match="Forbidden. You don't have permission to access this resource."): list(read_full_refresh(stream)) @@ -96,7 +91,7 @@ def test_dashboards_stream(config, dashboards_response): f"https://{config['domain']}/rest/api/3/dashboard", json=dashboards_response, ) - + stream = find_stream("dashboards", config) records = list(read_full_refresh(stream)) @@ -132,7 +127,7 @@ def test_groups_stream(config, groups_response): def test_issues_fields_stream(config, mock_fields_response): stream = find_stream("issue_fields", config) records = list(read_full_refresh(stream)) - + assert len(records) == 6 assert len(responses.calls) == 1 @@ -149,7 +144,7 @@ def test_python_issues_fields_ids_by_name(config, mock_fields_response): "Issue Type": ["issuetype"], "Parent": ["parent"], "Issue Type2": ["issuetype2"], - "Issue Type3": ["issuetype3"] + "Issue Type3": ["issuetype3"], } assert expected_ids_by_name == stream.field_ids_by_name() @@ -400,7 +395,9 @@ def test_screen_tabs_stream(config, mock_screen_response, screen_tabs_response): @responses.activate def test_sprints_stream(config, mock_board_response, mock_sprints_response): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprints", SyncMode.full_refresh).build() + ) assert len(output.records) == 3 assert len(responses.calls) == 4 @@ -417,7 +414,7 @@ def test_board_does_not_support_sprints(config, mock_board_response, sprints_res responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50", json={"errorMessages": ["The board does not support sprints"], "errors": {}}, - status=400 + status=400, ) responses.add( responses.GET, @@ -443,7 +440,11 @@ def test_sprint_issues_stream(config, mock_board_response, mock_fields_response, json=sprints_issues_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("sprint_issues", SyncMode.full_refresh).build(), + ) assert len(output.records) == 3 assert len(responses.calls) == 8 @@ -585,7 +586,7 @@ def test_avatars_stream_should_retry(config, caplog): responses.GET, f"https://{config['domain']}/rest/api/3/avatar/{slice}/system", json={"errorMessages": ["The error message"], "errors": {}}, - status=400 + status=400, ) stream = find_stream("avatars", config) @@ -622,9 +623,11 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project assert "non_empty_field" in records[0]["fields"] assert len(responses.calls) == 3 - error_message = ("Stream `issues`. An error occurred, details: The user doesn't have " - 'permission to the project. Please grant the user to the project. Errors: ' - '["The value \'3\' does not exist for the field \'project\'."]') + error_message = ( + "Stream `issues`. An error occurred, details: The user doesn't have " + "permission to the project. Please grant the user to the project. Errors: " + "[\"The value '3' does not exist for the field 'project'.\"]" + ) assert error_message in caplog.messages @@ -632,23 +635,24 @@ def test_python_issues_stream(config, mock_projects_responses_additional_project @pytest.mark.parametrize( "status_code, response_errorMessages, expected_log_message", ( - (400, - ["The value 'incorrect_project' does not exist for the field 'project'."], - ( - "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." - " Please grant the user to the project. " - "Errors: [\"The value \'incorrect_project\' does not exist for the field \'project\'.\"]" - ) - ), + ( + 400, + ["The value 'incorrect_project' does not exist for the field 'project'."], ( - 403, - ["The value 'incorrect_project' doesn't have permission for the field 'project'."], - ( - 'Stream `issues`. An error occurred, details:' - ' Errors: ["The value \'incorrect_project\' doesn\'t have permission for the field \'project\'."]' - ) + "Stream `issues`. An error occurred, details: The user doesn't have permission to the project." + " Please grant the user to the project. " + "Errors: [\"The value 'incorrect_project' does not exist for the field 'project'.\"]" ), - ) + ), + ( + 403, + ["The value 'incorrect_project' doesn't have permission for the field 'project'."], + ( + "Stream `issues`. An error occurred, details:" + " Errors: [\"The value 'incorrect_project' doesn't have permission for the field 'project'.\"]" + ), + ), + ), ) def test_python_issues_stream_skip_on_http_codes_error_handling(config, status_code, response_errorMessages, expected_log_message, caplog): responses.add( @@ -689,8 +693,7 @@ def test_python_issues_stream_updated_state(config): stream = Issues(**args) updated_state = stream._get_updated_state( - current_stream_state={"updated": "2021-01-01T00:00:00Z"}, - latest_record={"updated": "2021-01-02T00:00:00Z"} + current_stream_state={"updated": "2021-01-01T00:00:00Z"}, latest_record={"updated": "2021-01-02T00:00:00Z"} ) assert updated_state == {"updated": "2021-01-02T00:00:00Z"} @@ -703,7 +706,7 @@ def test_python_issues_stream_updated_state(config): ("pullrequest={dataType=pullrequest, state=thestate, stateCount=1}", True), ("pullrequest={dataType=pullrequest, state=thestate, stateCount=0}", False), ("{}", False), - ) + ), ) def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pull_request): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) @@ -721,7 +724,9 @@ def test_python_pull_requests_stream_has_pull_request(config, dev_field, has_pul @responses.activate -def test_python_pull_requests_stream_has_pull_request(config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter): +def test_python_pull_requests_stream_has_pull_request( + config, mock_fields_response, mock_projects_responses_additional_project, mock_issues_responses_with_date_filter +): authenticator = SourceJira(config=config, catalog=None, state=None).get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} issues_stream = Issues(**args) @@ -822,7 +827,11 @@ def test_project_permissions_stream(config, mock_non_deleted_projects_responses, @responses.activate def test_project_email_stream(config, mock_non_deleted_projects_responses, mock_project_emails): - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_email", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -836,7 +845,11 @@ def test_project_components_stream(config, mock_non_deleted_projects_responses, json=project_components_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("project_components", SyncMode.full_refresh).build(), + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -850,7 +863,11 @@ def test_permissions_stream(config, permissions_response): json=permissions_response, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), + config, + CatalogBuilder().with_stream("permissions", SyncMode.full_refresh).build(), + ) assert len(output.records) == 1 assert len(responses.calls) == 1 @@ -869,7 +886,9 @@ def test_labels_stream(config, labels_response): json={}, ) - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream("labels", SyncMode.full_refresh).build() + ) assert len(output.records) == 2 assert len(responses.calls) == 2 @@ -955,38 +974,33 @@ def test_project_versions_stream(config, mock_non_deleted_projects_responses, pr @pytest.mark.parametrize( "stream, expected_records_number, expected_calls_number, log_message", [ - ( - "issues", - 2, - 4, - "The user doesn't have permission to the project. Please grant the user to the project." - ), + ("issues", 2, 4, "The user doesn't have permission to the project. Please grant the user to the project."), ( "issue_custom_field_contexts", 2, 4, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_contexts`. An error occurred, details: ['Not found issue custom field context for issue fields issuetype2']. Skipping for now. ", ), ( "issue_custom_field_options", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_custom_field_options`. An error occurred, details: ['Not found issue custom field options for issue fields issuetype3']. Skipping for now. ", ), ( "issue_watchers", 1, 6, - "Not found. The requested resource was not found on the server." + "Not found. The requested resource was not found on the server.", # "Stream `issue_watchers`. An error occurred, details: ['Not found watchers for issue TESTKEY13-2']. Skipping for now. ", ), ( "project_email", 4, 4, - "Forbidden. You don't have permission to access this resource." + "Forbidden. You don't have permission to access this resource.", # "Stream `project_email`. An error occurred, details: ['No access to emails for project 3']. Skipping for now. ", ), ], @@ -1009,7 +1023,9 @@ def test_skip_slice( log_message, ): config["projects"] = config.get("projects", []) + ["Project3", "Project4"] - output = read(SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build()) + output = read( + SourceJira(config=config, catalog=None, state=None), config, CatalogBuilder().with_stream(stream, SyncMode.full_refresh).build() + ) assert len(output.records) == expected_records_number assert len(responses.calls) == expected_calls_number diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py index 8789782701546..a217637afe7d8 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/config.py @@ -5,7 +5,7 @@ class KlaviyoConfigBuilder: def __init__(self) -> None: - self._config = {"api_key":"an_api_key","start_date":"2021-01-01T00:00:00Z"} + self._config = {"api_key": "an_api_key", "start_date": "2021-01-01T00:00:00Z"} def with_start_date(self, start_date: datetime) -> "KlaviyoConfigBuilder": self._config["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py index 969306e0d75c2..fd77d33fb5096 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/integration/test_profiles.py @@ -37,11 +37,11 @@ def _a_profile_request(start_date: datetime) -> HttpRequest: return HttpRequest( url=f"https://a.klaviyo.com/api/profiles", query_params={ - "additional-fields[profile]": "predictive_analytics", - "page[size]": "100", - "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", - "sort": "updated" - } + "additional-fields[profile]": "predictive_analytics", + "page[size]": "100", + "filter": f"greater-than(updated,{start_date.strftime('%Y-%m-%dT%H:%M:%S%z')})", + "sort": "updated", + }, ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py index 211a158290e35..df84e66f80295 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_included_extractor.py @@ -34,26 +34,26 @@ def extractor(mock_config, mock_field_path, mock_decoder): return KlaviyoIncludedFieldExtractor(mock_field_path, mock_config, mock_decoder) -@patch('dpath.get') -@patch('dpath.values') +@patch("dpath.get") +@patch("dpath.values") def test_extract_records_by_path(mock_values, mock_get, extractor, mock_response, mock_decoder): - mock_values.return_value = [{'key': 'value'}] - mock_get.return_value = {'key': 'value'} - mock_decoder.decode.return_value = {'data': 'value'} + mock_values.return_value = [{"key": "value"}] + mock_get.return_value = {"key": "value"} + mock_decoder.decode.return_value = {"data": "value"} - field_paths = ['data'] + field_paths = ["data"] records = list(extractor.extract_records_by_path(mock_response, field_paths)) - assert records == [{'key': 'value'}] + assert records == [{"key": "value"}] mock_values.return_value = [] mock_get.return_value = None - records = list(extractor.extract_records_by_path(mock_response, ['included'])) + records = list(extractor.extract_records_by_path(mock_response, ["included"])) assert records == [] def test_update_target_records_with_included(extractor): - target_records = [{'relationships': {'type1': {'data': {'id': 1}}}}] - included_records = [{'id': 1, 'type': 'type1', 'attributes': {'key': 'value'}}] + target_records = [{"relationships": {"type1": {"data": {"id": 1}}}}] + included_records = [{"id": 1, "type": "type1", "attributes": {"key": "value"}}] updated_records = list(extractor.update_target_records_with_included(target_records, included_records)) - assert updated_records[0]['relationships']['type1']['data'] == {'id': 1, 'key': 'value'} + assert updated_records[0]["relationships"]["type1"]["data"] == {"id": 1, "key": "value"} diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py index 052a93ae0f30d..f236517b66d85 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_per_partition_state_migration.py @@ -26,14 +26,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): input_state = { "states": [ - { - "partition": {"parent_id": "13506132"}, - "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"} - }, - { - "partition": {"parent_id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, + {"partition": {"parent_id": "13506132"}, "cursor": {"last_changed": "2023-12-27T08:34:39+00:00"}}, + {"partition": {"parent_id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] } @@ -61,14 +55,10 @@ def _migrator(): parent_key="{{ parameters['parent_key_id'] }}", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ) - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py index a25db137a57c8..9561952f3e0f8 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py @@ -29,16 +29,12 @@ def _source() -> SourceKlaviyo: ( 400, False, - ( - "Bad request. Please check your request parameters." - ), + ("Bad request. Please check your request parameters."), ), ( 403, False, - ( - "Please provide a valid API key and make sure it has permissions to read specified streams." - ), + ("Please provide a valid API key and make sure it has permissions to read specified streams."), ), ), ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py index 2ed50d5e46f41..6c6bd0393ba8e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -36,6 +36,7 @@ EVENTS_STREAM_STATE_DATE = (datetime.fromisoformat(EVENTS_STREAM_CONFIG_START_DATE) + relativedelta(years=1)).isoformat() EVENTS_STREAM_TESTING_FREEZE_TIME = "2023-12-12 12:00:00" + def get_step_diff(provided_date: str) -> int: """ This function returns the difference in weeks between provided date and freeze time. @@ -44,6 +45,7 @@ def get_step_diff(provided_date: str) -> int: freeze_date = datetime.strptime(EVENTS_STREAM_TESTING_FREEZE_TIME, "%Y-%m-%d %H:%M:%S") return (freeze_date - provided_date).days // 7 + def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> Stream: source = SourceKlaviyo(CatalogBuilder().build(), KlaviyoConfigBuilder().build(), StateBuilder().build()) matches_by_name = [stream_config for stream_config in source.streams(config) if stream_config.name == stream_name] @@ -84,9 +86,7 @@ def path(self, **kwargs) -> str: class TestKlaviyoStream: def test_request_headers(self): stream = SomeStream(api_key=API_KEY) - expected_headers = { - "Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}" - } + expected_headers = {"Accept": "application/json", "Revision": stream.api_revision, "Authorization": f"Klaviyo-API-Key {API_KEY}"} assert stream.request_headers() == expected_headers @pytest.mark.parametrize( @@ -148,9 +148,7 @@ def test_availability_strategy(self): "This is most likely due to insufficient permissions on the credentials in use. " "Try to create and use an API key with read permission for the 'some_stream' stream granted" ) - reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes( - stream, None, None, None - ) + reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes(stream, None, None, None) assert expected_status_code in reasons_for_unavailable_status_codes assert reasons_for_unavailable_status_codes[expected_status_code] == expected_message @@ -173,9 +171,7 @@ def test_backoff_time_large_retry_after(self): response_mock.headers = {"Retry-After": retry_after} with pytest.raises(AirbyteTracedException) as e: stream.get_backoff_strategy().backoff_time(response_mock, _ANY_ATTEMPT_COUNT) - error_message = ( - "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." - ) + error_message = "Rate limit wait time 605.0 is greater than max waiting time of 600 seconds. Stopping the stream..." assert str(e.value) == error_message @@ -197,12 +193,12 @@ def generate_api_urls(start_date_str: str) -> list[(str, str)]: end_date = current_date start_date_str = start_date.strftime("%Y-%m-%dT%H:%M:%S") + start_date.strftime("%z") end_date_str = end_date.strftime("%Y-%m-%dT%H:%M:%S") + end_date.strftime("%z") - base_url = 'https://a.klaviyo.com/api/events' + base_url = "https://a.klaviyo.com/api/events" query_params = { - 'fields[metric]': 'name,created,updated,integration', - 'include': 'metric', - 'filter': f'greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})', - 'sort': 'datetime' + "fields[metric]": "name,created,updated,integration", + "include": "metric", + "filter": f"greater-or-equal(datetime,{start_date_str}),less-or-equal(datetime,{end_date_str})", + "sort": "datetime", } encoded_query = urllib.parse.urlencode(query_params) encoded_url = f"{base_url}?{encoded_query}" @@ -289,7 +285,6 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso latest_record={stream.cursor_field: latest_cursor}, ) == {stream.cursor_field: expected_cursor} - @freezegun.freeze_time("2023-12-12 12:00:00") @pytest.mark.parametrize( # expected_amount_of_results: we put 1 record for every request @@ -297,21 +292,17 @@ def test_get_updated_state(self, config_start_date, current_cursor, latest_curso ( ( # we pick the state - EVENTS_STREAM_CONFIG_START_DATE, - EVENTS_STREAM_STATE_DATE, - get_step_diff(EVENTS_STREAM_STATE_DATE) + 1 # adding last request + EVENTS_STREAM_CONFIG_START_DATE, + EVENTS_STREAM_STATE_DATE, + get_step_diff(EVENTS_STREAM_STATE_DATE) + 1, # adding last request ), ( - # we pick the config start date - EVENTS_STREAM_CONFIG_START_DATE, - None, - get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1 # adding last request - ), - ( - "", - "", - get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1 # adding last request + # we pick the config start date + EVENTS_STREAM_CONFIG_START_DATE, + None, + get_step_diff(EVENTS_STREAM_CONFIG_START_DATE) + 1, # adding last request ), + ("", "", get_step_diff(EVENTS_STREAM_DEFAULT_START_DATE) + 1), # adding last request ), ) def test_read_records_events(self, config_start_date, stream_state, expected_amount_of_results, requests_mock): @@ -370,9 +361,7 @@ class TestSemiIncrementalKlaviyoStream: ) def test_read_records(self, start_date, stream_state, input_records, expected_records, requests_mock): stream = get_stream_by_name("metrics", CONFIG | {"start_date": start_date}) - requests_mock.register_uri( - "GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records} - ) + requests_mock.register_uri("GET", f"https://a.klaviyo.com/api/metrics", status_code=200, json={"data": input_records}) stream.stream_state = {stream.cursor_field: stream_state if stream_state else start_date} records = get_records(stream=stream, sync_mode=SyncMode.incremental) assert records == expected_records @@ -617,9 +606,9 @@ def test_stream_slices(self): ) def test_request_params(self, stream_state, stream_slice, next_page_token, expected_params): stream = Campaigns(api_key=API_KEY) - assert stream.request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ) == expected_params + assert ( + stream.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) == expected_params + ) class TestCampaignsDetailedStream: @@ -647,7 +636,9 @@ def test_set_recipient_count_not_found(self, requests_mock): mocked_response.ok = False mocked_response.status_code = 404 mocked_response.json.return_value = {} - with patch.object(stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response)): + with patch.object( + stream._http_client, "send_request", return_value=(mock.MagicMock(spec=requests.PreparedRequest), mocked_response) + ): stream._set_recipient_count(record) assert record["estimated_recipient_count"] == 0 diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py index 58ff037abc679..9e53da81f4d23 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_bank_balances_stream.py @@ -20,10 +20,7 @@ def patch_base_class(mocker): def test_stream_slices(patch_base_class): stream = BankBalancesStream(**config()) - account_uuids = [ - {"account_uuid": "first"}, - {"account_uuid": "second"} - ] + account_uuids = [{"account_uuid": "first"}, {"account_uuid": "second"}] stream.get_account_uuids = MagicMock(return_value=account_uuids) stream.start_date = date(2022, 1, 1) stream.end_date = date(2022, 1, 2) @@ -43,7 +40,7 @@ def test_stream_slices(patch_base_class): { "account_uuid": "second", "date": "2022-01-02", - } + }, ] slices = stream.stream_slices() assert slices == expected diff --git a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py index 1bda3981cbd62..6faf061736f36 100644 --- a/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-kyriba/unit_tests/test_source.py @@ -13,6 +13,7 @@ "start_date": "2022-01-01", } + def test_check_connection(mocker): source = SourceKyriba() KyribaClient.login = MagicMock() diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py index 2ec43d5c80dbe..214e0fd31cfb6 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py @@ -39,7 +39,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: pools = config.get("pool_ids").split(",") start_ids = config.get("start_ids").split(",") - for (pool_id, start_id) in zip(pools, start_ids): + for pool_id, start_id in zip(pools, start_ids): response = requests.get(f"{config['url_base']}/kyve/query/v1beta1/pool/{pool_id}") pool_data = response.json().get("pool").get("data") diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py index 2fa7fbf0dd34f..bebaa94a46bad 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_components.py @@ -35,7 +35,13 @@ def mock_response(): @pytest.fixture def mock_analytics_cursor_params(): - return {"start_datetime": MagicMock(), "cursor_field": MagicMock(), "datetime_format": "%s", "config": MagicMock(), "parameters": MagicMock()} + return { + "start_datetime": MagicMock(), + "cursor_field": MagicMock(), + "datetime_format": "%s", + "config": MagicMock(), + "parameters": MagicMock(), + } @pytest.fixture diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py index 931da4bfbbcdf..ffb302290c824 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -90,7 +90,9 @@ def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> No @pytest.mark.parametrize("error_code", [429, 500, 503]) def test_should_retry_on_error(self, error_code, requests_mock, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) stream = find_stream("accounts", TEST_CONFIG) requests_mock.register_uri( @@ -102,11 +104,10 @@ def test_should_retry_on_error(self, error_code, requests_mock, mocker): def test_custom_streams(self): config = {"ad_analytics_reports": [{"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}], **TEST_CONFIG} - ad_campaign_analytics = find_stream('ad_campaign_analytics', config) + ad_campaign_analytics = find_stream("ad_campaign_analytics", config) for stream in self._instance._create_custom_ad_analytics_streams(config=config): assert isinstance(stream, type(ad_campaign_analytics)) - @pytest.mark.parametrize( "stream_name, expected", [ @@ -136,26 +137,23 @@ def test_path(self, stream_name, expected): @pytest.mark.parametrize( ("status_code", "is_connection_successful", "error_msg"), ( - ( - 400, - False, - ( - "Bad request. Please check your request parameters." - ), - ), - ( - 403, - False, - ( - "Forbidden. You don't have permission to access this resource." - ), - ), - (200, True, None), + ( + 400, + False, + ("Bad request. Please check your request parameters."), + ), + ( + 403, + False, + ("Forbidden. You don't have permission to access this resource."), + ), + (200, True, None), ), ) def test_check_connection(self, requests_mock, status_code, is_connection_successful, error_msg, mocker): - mocker.patch.object(ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", - side_effect=self._mock_initialize_cache_for_parent_streams) + mocker.patch.object( + ManifestDeclarativeSource, "_initialize_cache_for_parent_streams", side_effect=self._mock_initialize_cache_for_parent_streams + ) mocker.patch("time.sleep", lambda x: None) json = {"elements": [{"data": []}] * 500} if 200 >= status_code < 300 else {} requests_mock.register_uri( diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py index 197f43ea8e648..d43199f5e9c55 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_streams.py @@ -50,7 +50,8 @@ def test_read_records(requests_mock): requests_mock.get("https://api.linkedin.com/rest/adAccounts", json={"elements": [{"id": 1}]}) requests_mock.get( "https://api.linkedin.com/rest/adAccounts/1/adCampaigns?q=search&search=(status:(values:List(ACTIVE,PAUSED,ARCHIVED,COMPLETED,CANCELED,DRAFT,PENDING_DELETION,REMOVED)))", - json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}) + json={"elements": [{"id": 1111, "lastModified": "2021-01-15"}]}, + ) requests_mock.get( "https://api.linkedin.com/rest/adAnalytics", [ diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py index e116afe2bbbdd..e6f0d5b881f7d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -65,7 +65,7 @@ } }, "pivot": "TEST_PIVOT_VALUE", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] @@ -144,6 +144,6 @@ "end_date": "2021-08-13", "_pivot": "TEST_PIVOT_VALUE", "string_of_pivot_values": "TEST_PIVOT_VALUE_1,TEST_PIVOT_VALUE_2", - "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"] + "pivotValues": ["TEST_PIVOT_VALUE_1", "TEST_PIVOT_VALUE_2"], } ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py index 73be5f6a05645..c62676cfc0ff3 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/test_update_specific_key.py @@ -74,8 +74,8 @@ "nested_dictionary_update", "list_of_dictionaries_update", "excluded_key_in_nested_dict", - "nested_list_with_mixed_types" - ] + "nested_list_with_mixed_types", + ], ) def test_update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys, expected_output): result = update_specific_key(target_dict, target_key, target_value, condition_func, excluded_keys) diff --git a/airbyte-integrations/connectors/source-looker/source_looker/components.py b/airbyte-integrations/connectors/source-looker/source_looker/components.py index d30b120a5e971..b69e09c0de2d9 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/components.py +++ b/airbyte-integrations/connectors/source-looker/source_looker/components.py @@ -14,7 +14,6 @@ @dataclass class LookerAuthenticator(NoAuth): - """ Authenticator that sets the Authorization header on the HTTP requests sent using access token which is updated upon expiration. diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py index f088ce69b9fca..3f174cc117e47 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py @@ -98,9 +98,7 @@ def fake_records_gen(): def get_stream_by_name(stream_name: str, config: Mapping[str, Any]) -> DeclarativeStream: source = SourceMarketo() - matches_by_name = [ - stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name - ] + matches_by_name = [stream_config for stream_config in source._get_declarative_streams(config) if stream_config.name == stream_name] if not matches_by_name: raise ValueError("Please provide a valid stream name.") return matches_by_name[0] diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index d77625438bbc0..647b4caa93ede 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -38,12 +38,7 @@ def test_should_retry_quota_exceeded(config, requests_mock): response_json = { "requestId": "d2ca#18c0b9833bf", "success": False, - "errors": [ - { - "code": "1029", - "message": "Export daily quota 500MB exceeded." - } - ] + "errors": [{"code": "1029", "message": "Export daily quota 500MB exceeded."}], } requests_mock.register_uri("GET", create_job_url, status_code=200, json=response_json) @@ -132,8 +127,8 @@ def test_activities_schema(activity, expected_schema, config): ( ( "Campaign Run ID,Choice Number,Has Predictive,Step ID,Test Variant,attributes\n" - "1,3,true,10,15,{\"spam\": \"true\"}\n" - "2,3,false,11,16,{\"spam\": \"false\"}" + '1,3,true,10,15,{"spam": "true"}\n' + '2,3,false,11,16,{"spam": "false"}' ), [ { @@ -231,9 +226,7 @@ def test_parse_response_incremental(config, requests_mock): created_at_record_1 = START_DATE.add(days=1).strftime("%Y-%m-%dT%H:%M:%SZ") created_at_record_2 = START_DATE.add(days=3).strftime("%Y-%m-%dT%H:%M:%SZ") current_state = START_DATE.add(days=2).strftime("%Y-%m-%dT%H:%M:%SZ") - response = { - "result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}] - } + response = {"result": [{"id": "1", "createdAt": created_at_record_1}, {"id": "2", "createdAt": created_at_record_2}]} requests_mock.get("/rest/v1/campaigns.json", json=response) stream = get_stream_by_name("campaigns", config) @@ -322,16 +315,8 @@ def test_get_updated_state(config, latest_record, current_state, expected_state) def test_filter_null_bytes(config): stream = Leads(config) - test_lines = [ - "Hello\x00World\n", - "Name,Email\n", - "John\x00Doe,john.doe@example.com\n" - ] - expected_lines = [ - "HelloWorld\n", - "Name,Email\n", - "JohnDoe,john.doe@example.com\n" - ] + test_lines = ["Hello\x00World\n", "Name,Email\n", "John\x00Doe,john.doe@example.com\n"] + expected_lines = ["HelloWorld\n", "Name,Email\n", "JohnDoe,john.doe@example.com\n"] filtered_lines = stream.filter_null_bytes(test_lines) for expected_line, filtered_line in zip(expected_lines, filtered_lines): assert expected_line == filtered_line @@ -340,15 +325,8 @@ def test_filter_null_bytes(config): def test_csv_rows(config): stream = Leads(config) - test_lines = [ - "Name,Email\n", - "John Doe,john.doe@example.com\n", - "Jane Doe,jane.doe@example.com\n" - ] - expected_records = [ - {"Name": "John Doe", "Email": "john.doe@example.com"}, - {"Name": "Jane Doe", "Email": "jane.doe@example.com"} - ] + test_lines = ["Name,Email\n", "John Doe,john.doe@example.com\n", "Jane Doe,jane.doe@example.com\n"] + expected_records = [{"Name": "John Doe", "Email": "john.doe@example.com"}, {"Name": "Jane Doe", "Email": "jane.doe@example.com"}] records = stream.csv_rows(test_lines) for expected_record, record in zip(expected_records, records): assert expected_record == record diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py index e67edc6d9fa50..6b9cb9cdb9cdc 100644 --- a/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py @@ -275,9 +275,11 @@ def test_get_shared_drive_object( ): mock_get_access_token.return_value = "dummy_access_token" mock_responses = [ - initial_response - if isinstance(initial_response, MagicMock) - else MagicMock(status_code=200, json=MagicMock(return_value=initial_response)) + ( + initial_response + if isinstance(initial_response, MagicMock) + else MagicMock(status_code=200, json=MagicMock(return_value=initial_response)) + ) ] for response in subsequent_responses: mock_responses.append( diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py index 205e1399c52c9..b70e7c43e5f7d 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_stream_reader.py @@ -435,9 +435,11 @@ def test_get_shared_drive_object( ): mock_get_access_token.return_value = "dummy_access_token" mock_responses = [ - initial_response - if isinstance(initial_response, MagicMock) - else MagicMock(status_code=200, json=MagicMock(return_value=initial_response)) + ( + initial_response + if isinstance(initial_response, MagicMock) + else MagicMock(status_code=200, json=MagicMock(return_value=initial_response)) + ) ] for response in subsequent_responses: mock_responses.append( @@ -465,9 +467,10 @@ def test_get_shared_drive_object( ], ) def test_drives_property(auth_type, user_principal_name, has_refresh_token): - with patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, patch( - "source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client" - ) as mock_one_drive_client: + with ( + patch("source_microsoft_sharepoint.stream_reader.execute_query_with_retry") as mock_execute_query, + patch("source_microsoft_sharepoint.stream_reader.SourceMicrosoftSharePointStreamReader.one_drive_client") as mock_one_drive_client, + ): refresh_token = "dummy_refresh_token" if has_refresh_token else None # Setup for different authentication types config_mock = MagicMock( diff --git a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py index 5e88631a341fb..79b035dee07f7 100644 --- a/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py @@ -49,9 +49,10 @@ def test_execute_query_with_retry(status_code, retry_after_header, expected_retr obj = Mock() obj.execute_query = Mock(side_effect=MockException(status_code, {"Retry-After": retry_after_header})) - with patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, patch( - "source_microsoft_sharepoint.utils.datetime" - ) as mock_datetime: + with ( + patch("source_microsoft_sharepoint.utils.time.sleep") as mock_sleep, + patch("source_microsoft_sharepoint.utils.datetime") as mock_datetime, + ): start_time = datetime(2021, 1, 1, 0, 0, 0) if retry_after_header: mock_datetime.now.side_effect = [start_time] * 2 + [ diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py index 69c842e5e2559..1df5c3b847c12 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py @@ -45,12 +45,14 @@ def patch_time(mocker): ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH + def delete_cache_files(cache_directory): directory_path = Path(cache_directory) if directory_path.exists() and directory_path.is_dir(): for file_path in directory_path.glob("*.sqlite"): file_path.unlink() + @pytest.fixture(autouse=True) def clear_cache_before_each_test(): # The problem: Once the first request is cached, we will keep getting the cached result no matter what setup we prepared for a particular test. diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index f877bdd997791..e1272e7d22c0a 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -22,12 +22,7 @@ def check_connection_url(config): return get_url_to_mock(export_stream) -@pytest.mark.parametrize( - "response_code,expect_success,response_json", - [ - (400, False, {"error": "Request error"}) - ] -) +@pytest.mark.parametrize("response_code,expect_success,response_json", [(400, False, {"error": "Request error"})]) def test_check_connection(requests_mock, check_connection_url, config_raw, response_code, expect_success, response_json): # requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, response_json)) requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=response_code, json=response_json) @@ -135,7 +130,7 @@ def test_streams_string_date(requests_mock, config_raw): "select_properties_by_default": True, "region": "EU", "date_window_size": 10, - "page_size": 1000 + "page_size": 1000, }, True, None, @@ -143,9 +138,9 @@ def test_streams_string_date(requests_mock, config_raw): ), ) def test_config_validation(config, success, expected_error_message, requests_mock): - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) - requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{'a': 1, 'created':'2021-02-11T00:00:00Z'}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) + requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json=[{"a": 1, "created": "2021-02-11T00:00:00Z"}]) try: is_success, message = SourceMixpanel().check_connection(None, config) except AirbyteTracedException as e: diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index 6f64dbf39aa50..a5ce2036ff6d8 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -95,7 +95,7 @@ def cohorts_response(): ) -def init_stream(name='', config=None): +def init_stream(name="", config=None): streams = SourceMixpanel().streams(config) for stream in streams: if stream.name == name: @@ -104,10 +104,10 @@ def init_stream(name='', config=None): def test_cohorts_stream_incremental(requests_mock, cohorts_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - config_raw['start_date'] = '2022-01-01T00:00:00Z' + config_raw["start_date"] = "2022-01-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", cohorts_response) - cohorts_stream = init_stream('cohorts', config=config_raw) + cohorts_stream = init_stream("cohorts", config=config_raw) records = read_incremental(cohorts_stream, stream_state={"created": "2022-04-19 23:22:01"}, cursor_field=["created"]) @@ -158,25 +158,14 @@ def engage_response(): def test_engage_stream_incremental(requests_mock, engage_response, config_raw): """Filter 1 old value, 1 new record should be returned""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2022-02-01T00:00:00Z' - config_raw['end_date'] = '2024-05-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2022-02-01T00:00:00Z" + config_raw["end_date"] = "2024-05-01T00:00:00Z" requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) stream_state = {"last_seen": "2024-02-11T11:20:47"} records = list(read_incremental(stream, stream_state=stream_state, cursor_field=["last_seen"])) @@ -193,97 +182,97 @@ def test_engage_stream_incremental(requests_mock, engage_response, config_raw): {}, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "abnormal_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 0, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2030-01-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2030-01-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "medium_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 1, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "early_state", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, } ] - } + }, ), ( "state_for_different_partition", { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, } ] }, 2, { - 'states': [ + "states": [ { - 'cursor': {'last_seen': '2024-02-01T00:00:00'}, - 'partition': {'id': 2222, 'parent_slice': {}}, + "cursor": {"last_seen": "2024-02-01T00:00:00"}, + "partition": {"id": 2222, "parent_slice": {}}, }, { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 1111, 'parent_slice': {}}, - } + "cursor": {"last_seen": "2024-03-01T11:20:47"}, + "partition": {"id": 1111, "parent_slice": {}}, + }, ] - } + }, ), ), ) @@ -291,26 +280,17 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi """Cohort_members stream has legacy state but actually it should always return all records because members in cohorts can be updated at any time """ - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{'id': 1111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}]) + requests_mock.register_uri( + "GET", MIXPANEL_BASE_URL + "cohorts/list", json=[{"id": 1111, "name": "bla", "created": "2024-02-02T00:00:00Z"}] + ) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?", engage_response) - stream = init_stream('cohort_members', config=config_raw) + stream = init_stream("cohort_members", config=config_raw) records = list(read_incremental(stream, stream_state=state, cursor_field=["last_seen"])) @@ -321,99 +301,85 @@ def test_cohort_members_stream_incremental(requests_mock, engage_response, confi def test_cohort_members_stream_pagination(requests_mock, engage_response, config_raw): """Cohort_members pagination""" - engage_properties = { - "results": { - "$browser": { - "count": 124, - "type": "string" - }, - "$browser_version": { - "count": 124, - "type": "string" - } - } - } - config_raw['start_date'] = '2024-02-01T00:00:00Z' - config_raw['end_date'] = '2024-03-01T00:00:00Z' - - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "cohorts/list", json=[ - {'id': 71000, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 71111, "name":'bla', 'created': '2024-02-02T00:00:00Z'}, - {'id': 72222, "name":'bla', 'created': '2024-02-01T00:00:00Z'}, - {'id': 73333, "name":'bla', 'created': '2024-02-03T00:00:00Z'}, - ]) - requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage", [ - { # initial request for 71000 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - }, - { # initial request for 71111 cohort and further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 2002, - "results": [ - { - "$distinct_id": "71111_1", - "$properties": { - "$created": "2024-03-01T11:20:47", - "$last_seen": "2024-03-01T11:20:47", + engage_properties = {"results": {"$browser": {"count": 124, "type": "string"}, "$browser_version": {"count": 124, "type": "string"}}} + config_raw["start_date"] = "2024-02-01T00:00:00Z" + config_raw["end_date"] = "2024-03-01T00:00:00Z" + requests_mock.register_uri( + "GET", + MIXPANEL_BASE_URL + "cohorts/list", + json=[ + {"id": 71000, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 71111, "name": "bla", "created": "2024-02-02T00:00:00Z"}, + {"id": 72222, "name": "bla", "created": "2024-02-01T00:00:00Z"}, + {"id": 73333, "name": "bla", "created": "2024-02-03T00:00:00Z"}, + ], + ) + requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "engage/properties", json=engage_properties) + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage", + [ + { # initial request for 71000 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + { # initial request for 71111 cohort and further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 2002, + "results": [ + { + "$distinct_id": "71111_1", + "$properties": { + "$created": "2024-03-01T11:20:47", + "$last_seen": "2024-03-01T11:20:47", + }, }, - }, - { - "$distinct_id": "71111_2", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", - } - } - ] - } - }, { # initial request for 72222 cohort without further pagination - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 1, - "results": [ - { - "$distinct_id": "72222_1", - "$properties": { - "$created": "2024-02-01T11:20:47", - "$last_seen": "2024-02-01T11:20:47", + { + "$distinct_id": "71111_2", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, + }, + ], + }, + }, + { # initial request for 72222 cohort without further pagination + "status_code": 200, + "json": { + "page": 0, + "page_size": 1000, + "session_id": "1234567890", + "status": "ok", + "total": 1, + "results": [ + { + "$distinct_id": "72222_1", + "$properties": { + "$created": "2024-02-01T11:20:47", + "$last_seen": "2024-02-01T11:20:47", + }, } - } - ] - } - },{ # initial request for 73333 cohort - 'status_code': 200, - 'json': { - "page": 0, - "page_size": 1000, - "session_id": "1234567890", - "status": "ok", - "total": 0, - "results": [] - } - } - ] + ], + }, + }, + { # initial request for 73333 cohort + "status_code": 200, + "json": {"page": 0, "page_size": 1000, "session_id": "1234567890", "status": "ok", "total": 0, "results": []}, + }, + ], ) # request for 1 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=1", + json={ "page": 1, "session_id": "1234567890", "status": "ok", @@ -423,13 +389,16 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) # request for 2 page for 71111 cohort - requests_mock.register_uri("POST", MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", json={ + requests_mock.register_uri( + "POST", + MIXPANEL_BASE_URL + "engage?page_size=1000&session_id=1234567890&page=2", + json={ "page": 2, "session_id": "1234567890", "status": "ok", @@ -439,27 +408,23 @@ def test_cohort_members_stream_pagination(requests_mock, engage_response, config "$properties": { "$created": "2024-02-01T11:20:47", "$last_seen": "2024-02-01T11:20:47", - } + }, } - ] - } + ], + }, ) - stream = init_stream('cohort_members', config=config_raw) - + stream = init_stream("cohort_members", config=config_raw) + records = list(read_incremental(stream, stream_state={}, cursor_field=["last_seen"])) assert len(records) == 5 new_updated_state = stream.get_updated_state(current_stream_state={}, latest_record=records[-1] if records else None) - assert new_updated_state == {'states': [ - { - 'cursor': {'last_seen': '2024-03-01T11:20:47'}, - 'partition': {'id': 71111, 'parent_slice': {}} - }, - { - 'cursor': {'last_seen': '2024-02-01T11:20:47'}, - 'partition': {'id': 72222, 'parent_slice': {}} - } - ]} + assert new_updated_state == { + "states": [ + {"cursor": {"last_seen": "2024-03-01T11:20:47"}, "partition": {"id": 71111, "parent_slice": {}}}, + {"cursor": {"last_seen": "2024-02-01T11:20:47"}, "partition": {"id": 72222, "parent_slice": {}}}, + ] + } @pytest.fixture @@ -493,37 +458,30 @@ def funnels_response(start_date): }, ) + @pytest.fixture def funnel_ids_response(start_date): - return setup_response( - 200, - [{ - "funnel_id": 36152117, - "name": "test" - }] - ) + return setup_response(200, [{"funnel_id": 36152117, "name": "test"}]) def test_funnels_stream(requests_mock, config, funnels_response, funnel_ids_response, config_raw): config_raw["start_date"] = "2024-01-01T00:00:00Z" config_raw["end_date"] = "2024-04-01T00:00:00Z" - stream = init_stream('funnels', config=config_raw) + stream = init_stream("funnels", config=config_raw) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels/list", funnel_ids_response) requests_mock.register_uri("GET", MIXPANEL_BASE_URL + "funnels", funnels_response) stream_slices = list(stream.stream_slices(sync_mode=SyncMode.incremental)) assert len(stream_slices) > 3 - assert { - "funnel_id": stream_slices[0]['funnel_id'], - "name": stream_slices[0]['funnel_name'] - } == { + assert {"funnel_id": stream_slices[0]["funnel_id"], "name": stream_slices[0]["funnel_name"]} == { "funnel_id": "36152117", - "name": "test" + "name": "test", } records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices[0]) records = list(records) assert len(records) == 2 + @pytest.fixture def engage_schema_response(): return setup_response( @@ -552,7 +510,7 @@ def _minimize_schema(fill_schema, schema_original): def test_engage_schema(requests_mock, engage_schema_response, config_raw): - stream = init_stream('engage', config=config_raw) + stream = init_stream("engage", config=config_raw) requests_mock.register_uri("GET", get_url_to_mock(EngageSchema(authenticator=MagicMock(), **config_raw)), engage_schema_response) type_schema = {} _minimize_schema(type_schema, stream.get_json_schema()) @@ -600,7 +558,7 @@ def test_update_engage_schema(requests_mock, config, config_raw): }, ), ) - engage_stream = init_stream('engage', config=config_raw) + engage_stream = init_stream("engage", config=config_raw) engage_schema = engage_stream.get_json_schema() assert "someNewSchemaField" in engage_schema["properties"] @@ -619,13 +577,10 @@ def annotations_response(): def test_annotations_stream(requests_mock, annotations_response, config_raw): - stream = init_stream('annotations', config=config_raw) + stream = init_stream("annotations", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", annotations_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -648,14 +603,13 @@ def revenue_response(): "status": "ok", }, ) + + def test_revenue_stream(requests_mock, revenue_response, config_raw): - stream = init_stream('revenue', config=config_raw) + stream = init_stream("revenue", config=config_raw) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", revenue_response) - stream_slice = StreamSlice(partition={}, cursor_slice= { - "start_time": "2021-01-25", - "end_time": "2021-07-25" - }) + stream_slice = StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-25", "end_time": "2021-07-25"}) # read records for single slice records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) records = list(records) @@ -684,6 +638,7 @@ def test_export_schema(requests_mock, export_schema_response, config): records_length = sum(1 for _ in records) assert records_length == 2 + def test_export_get_json_schema(requests_mock, export_schema_response, config): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", export_schema_response) @@ -691,7 +646,7 @@ def test_export_get_json_schema(requests_mock, export_schema_response, config): stream = Export(authenticator=MagicMock(), **config) schema = stream.get_json_schema() - assert "DYNAMIC_FIELD" in schema['properties'] + assert "DYNAMIC_FIELD" in schema["properties"] @pytest.fixture @@ -728,6 +683,7 @@ def test_export_stream(requests_mock, export_response, config): records_length = sum(1 for _ in records) assert records_length == 1 + def test_export_stream_fail(requests_mock, export_response, config): stream = Export(authenticator=MagicMock(), **config) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py index 5b08cd7892447..4c2903b76df9b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/utils.py @@ -35,7 +35,9 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, stream_instance.state = stream_state slices = stream_instance.stream_slices(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_state=stream_state) for slice in slices: - records = stream_instance.read_records(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state) + records = stream_instance.read_records( + sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_slice=slice, stream_state=stream_state + ) for record in records: stream_state = stream_instance.get_updated_state(stream_state, record) res.append(record) diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py index 3dd017d476b78..b25b2c27ad98f 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_requests/base_requests_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class MondayBaseRequestBuilder(MondayRequestBuilder): diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py index 779d64d80af7c..2457b51faac00 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/integrations/monday_responses/error_response_builder.py @@ -19,4 +19,3 @@ def build(self, file_path: Optional[str] = None) -> HttpResponse: if not file_path: return HttpResponse(json.dumps(find_template(str(self._status_code), __file__)), self._status_code) return HttpResponse(json.dumps(find_template(str(file_path), __file__)), self._status_code) - diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index 2037f13ee02a1..a33e5866bb157 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -151,16 +151,18 @@ def test_build_items_incremental_query(monday_requester): "text": {"type": ["null", "string"]}, "type": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}, - "display_value": {"type": ["null", "string"]} + "display_value": {"type": ["null", "string"]}, } - } + }, } stream_slice = {"ids": [1, 2, 3]} built_query = monday_requester._build_items_incremental_query(object_name, field_schema, stream_slice) - assert built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," \ - "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + assert ( + built_query == "items(limit:100,ids:[1, 2, 3]){id,name,column_values{id,text,type,value,... on MirrorValue{display_value}," + "... on BoardRelationValue{display_value},... on DependencyValue{display_value}}}" + ) def test_get_request_headers(monday_requester): diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py index 488bcf605f2ec..06f90b4637351 100755 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/hook.py @@ -22,12 +22,13 @@ abnormal_state_write_file = support_file_path_prefix + "/temp/abnormal_state_copy.json" abnormal_state_file = support_file_path_prefix + "/abnormal_state_template.json" -secret_config_file = '/connector/secrets/cat-config.json' -secret_active_config_file = support_file_path_prefix + '/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/cat-config-cdc.json' -secret_active_config_cdc_file = support_file_path_prefix + '/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/cat-config.json" +secret_active_config_file = support_file_path_prefix + "/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/cat-config-cdc.json" +secret_active_config_cdc_file = support_file_path_prefix + "/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') @contextmanager def connect_to_db(): @@ -36,11 +37,7 @@ def connect_to_db(): conn = None try: conn = mysql.connector.connect( - database=None, - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + database=None, user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") yield conn @@ -54,6 +51,7 @@ def connect_to_db(): conn.close() print("Database connection closed") + def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: insert_query = f"INSERT INTO {schema_name}.{table_name} (id, name) VALUES (%s, %s) ON DUPLICATE KEY UPDATE id=id" try: @@ -66,6 +64,7 @@ def insert_records(conn, schema_name: str, table_name: str, records: List[Tuple[ print(f"Error inserting records: {error}") conn.rollback() + def create_schema(conn, schema_name: str) -> None: create_schema_query = f"CREATE DATABASE IF NOT EXISTS {schema_name}" try: @@ -77,32 +76,34 @@ def create_schema(conn, schema_name: str) -> None: print(f"Error creating database: {error}") conn.rollback() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path(support_file_path_prefix + "/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: secret = json.load(base_config) secret["database"] = schema_name - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn, schema_name: str, table_name: str) -> None: create_table_query = f""" CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( @@ -119,45 +120,44 @@ def create_table(conn, schema_name: str, table_name: str) -> None: print(f"Error creating table: {error}") conn.rollback() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] - table_name = 'id_and_name_cat' + new_records = [("4", "four"), ("5", "five")] + table_name = "id_and_name_cat" with connect_to_db() as conn: insert_records(conn, schema_name, table_name, new_records) + def setup(): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] with connect_to_db() as conn: create_schema(conn, schema_name) create_table(conn, schema_name, table_name) insert_records(conn, schema_name, table_name, records) + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_schemas_with_prefix(conn, date_prefix): query = f""" SELECT schema_name @@ -177,19 +177,22 @@ def delete_schemas_with_prefix(conn, date_prefix): print(f"An error occurred in deleting schema: {e}") sys.exit(1) + def teardown() -> None: today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") with connect_to_db() as conn: delete_schemas_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: schema_name = load_schema_name_from_catalog() print(f"delete database {schema_name}") with connect_to_db() as conn: delete_schemas_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py index 6f59654861817..be48cadc1a219 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_components.py @@ -8,52 +8,79 @@ def test_users_stream_transformation(): input_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "user": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } output_record = { - "object": "user", "id": "123", "name": "Airbyte", "avatar_url": "some url", "type": "bot", - "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", - "person": {"email": "email"}}}, "workspace_name": "test"} + "object": "user", + "id": "123", + "name": "Airbyte", + "avatar_url": "some url", + "type": "bot", + "bot": { + "owner": { + "type": "user", + "info": { + "object": "user", + "id": "id", + "name": "Test User", + "avatar_url": None, + "type": "person", + "person": {"email": "email"}, + }, + }, + "workspace_name": "test", + }, } assert NotionUserTransformation().transform(input_record) == output_record def test_notion_properties_transformation(): input_record = { - "id": "123", "properties": { - "Due date": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - }, + "id": "123", + "properties": { + "Due date": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, "Status": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, } output_record = { - "id": "123", "properties": [ + "id": "123", + "properties": [ { - "name": "Due date", "value": { - "id": "M%3BBw", "type": "date", "date": { - "start": "2023-02-23", "end": None, "time_zone": None - } - } + "name": "Due date", + "value": {"id": "M%3BBw", "type": "date", "date": {"start": "2023-02-23", "end": None, "time_zone": None}}, }, { "name": "Status", "value": { - "id": "Z%3ClH", "type": "status", "status": { - "id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default" - } - } - } - ] + "id": "Z%3ClH", + "type": "status", + "status": {"id": "86ddb6ec-0627-47f8-800d-b65afd28be13", "name": "Not started", "color": "default"}, + }, + }, + ], } assert NotionPropertiesTransformation().transform(input_record) == output_record @@ -64,55 +91,38 @@ def test_notion_properties_transformation(): {"id": "3", "last_edited_time": "2022-01-04T00:00:00.000Z"}, ] + @pytest.fixture def data_feed_config(): return NotionDataFeedFilter(parameters={}, config={"start_date": "2021-01-01T00:00:00.000Z"}) + @pytest.mark.parametrize( "state_value, expected_return", [ - ( - "2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z" - ), - ( - "2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z" - ), - ( - {}, "2021-01-01T00:00:00.000Z" - ) + ("2021-02-01T00:00:00.000Z", "2021-02-01T00:00:00.000Z"), + ("2020-01-01T00:00:00.000Z", "2021-01-01T00:00:00.000Z"), + ({}, "2021-01-01T00:00:00.000Z"), ], - ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"] + ids=["State value is greater than start_date", "State value is less than start_date", "Empty state, default to start_date"], ) def test_data_feed_get_filter_date(data_feed_config, state_value, expected_return): start_date = data_feed_config.config["start_date"] - + result = data_feed_config._get_filter_date(start_date, state_value) assert result == expected_return, f"Expected {expected_return}, but got {result}." -@pytest.mark.parametrize("stream_state,stream_slice,expected_records", [ - ( - {"last_edited_time": "2022-01-01T00:00:00.000Z"}, - {"id": "some_id"}, - state_test_records - ), - ( - {"last_edited_time": "2022-01-03T00:00:00.000Z"}, - {"id": "some_id"}, - [state_test_records[-2], state_test_records[-1]] - ), - ( - {"last_edited_time": "2022-01-05T00:00:00.000Z"}, - {"id": "some_id"}, - [] - ), - ( - {}, - {"id": "some_id"}, - state_test_records - ) -], -ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"]) +@pytest.mark.parametrize( + "stream_state,stream_slice,expected_records", + [ + ({"last_edited_time": "2022-01-01T00:00:00.000Z"}, {"id": "some_id"}, state_test_records), + ({"last_edited_time": "2022-01-03T00:00:00.000Z"}, {"id": "some_id"}, [state_test_records[-2], state_test_records[-1]]), + ({"last_edited_time": "2022-01-05T00:00:00.000Z"}, {"id": "some_id"}, []), + ({}, {"id": "some_id"}, state_test_records), + ], + ids=["No records filtered", "Some records filtered", "All records filtered", "Empty state: no records filtered"], +) def test_data_feed_filter_records(data_feed_config, stream_state, stream_slice, expected_records): filtered_records = data_feed_config.filter_records(state_test_records, stream_state, stream_slice) assert filtered_records == expected_records, "Filtered records do not match the expected records." diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py index 7b323ed72257b..73c30eb2ef6d8 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_python_streams.py @@ -79,11 +79,7 @@ def test_http_method(patch_base_class): @pytest.mark.parametrize( "response_json, expected_output", - [ - ({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), - ({"has_more": False}, None), - ({}, None) - ], + [({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), ({"has_more": False}, None), ({}, None)], ids=["Next_page_token exists with cursor", "No next_page_token", "No next_page_token"], ) def test_next_page_token(patch_base_class, response_json, expected_output): @@ -454,21 +450,149 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, def test_block_record_transformation(): stream = Blocks(parent=None, config=MagicMock()) response_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, - "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "page": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } expected_record = { - "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", - "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", - "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, - {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, - {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, - {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], - "color": "default"} + "object": "block", + "id": "id", + "parent": {"type": "page_id", "page_id": "id"}, + "created_time": "2021-10-19T13:33:00.000Z", + "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, + "last_edited_by": {"object": "user", "id": "id"}, + "has_children": False, + "archived": False, + "type": "paragraph", + "paragraph": { + "rich_text": [ + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "text", + "text": {"content": "@", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": True, + "color": "default", + }, + "plain_text": "@", + "href": None, + }, + { + "type": "text", + "text": {"content": "test", "link": None}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": None, + }, + { + "type": "mention", + "mention": {"type": "page", "info": {"id": "id"}}, + "annotations": { + "bold": False, + "italic": False, + "strikethrough": False, + "underline": False, + "code": False, + "color": "default", + }, + "plain_text": "test", + "href": "https://www.notion.so/id", + }, + ], + "color": "default", + }, } assert stream.transform(response_record) == expected_record diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py index c270f0894e5e4..3f15b354b6f9e 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -29,8 +29,7 @@ def test_get_authenticator(config, expected_token): def test_streams(): source = SourceNotion() - config_mock = {"start_date": "2020-01-01T00:00:00.000Z", - "credentials": {"auth_type": "token", "token": "abcd"}} + config_mock = {"start_date": "2020-01-01T00:00:00.000Z", "credentials": {"auth_type": "token", "token": "abcd"}} streams = source.streams(config_mock) expected_streams_number = 5 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-orb/components.py b/airbyte-integrations/connectors/source-orb/components.py index 3732d1f5954ac..b535063e5731f 100644 --- a/airbyte-integrations/connectors/source-orb/components.py +++ b/airbyte-integrations/connectors/source-orb/components.py @@ -63,7 +63,6 @@ class SubscriptionUsagePartitionRouter(StreamSlicer): config: Config def stream_slices(self) -> Iterable[StreamSlice]: - """ This stream is sliced per `subscription_id` and day, as well as `billable_metric_id` if a grouping key is provided. This is because the API only supports a diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 07d75f4bc2811..8bd4320c7e52b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -13,6 +13,7 @@ WARNING: Do not modify this file. """ + # Declarative Source class SourcePaypalTransaction(YamlDeclarativeSource): def __init__(self): diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py index dd19b6306e770..b277936f6bf13 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/auth_components_test.py @@ -16,49 +16,51 @@ def mock_authenticator(): return PayPalOauth2Authenticator( config={}, parameters={}, - client_id='test_client_id', - client_secret='test_client_secret', - token_refresh_endpoint='https://test.token.endpoint', - grant_type='test_grant_type' + client_id="test_client_id", + client_secret="test_client_secret", + token_refresh_endpoint="https://test.token.endpoint", + grant_type="test_grant_type", ) + def test_get_refresh_access_token_response(mock_authenticator): - expected_response_json = {'access_token': 'test_access_token', 'expires_in': 3600} + expected_response_json = {"access_token": "test_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=expected_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=expected_response_json, status_code=200) # Call _get_refresh method mock_authenticator._get_refresh_access_token_response() - - assert mock_authenticator.access_token == expected_response_json['access_token'] + + assert mock_authenticator.access_token == expected_response_json["access_token"] + def test_token_expiration(mock_authenticator): # Mock response for initial token request - initial_response_json = {'access_token': 'initial_access_token', 'expires_in': 1} + initial_response_json = {"access_token": "initial_access_token", "expires_in": 1} # Mock response for token refresh request - refresh_response_json = {'access_token': 'refreshed_access_token', 'expires_in': 3600} + refresh_response_json = {"access_token": "refreshed_access_token", "expires_in": 3600} with requests_mock.Mocker() as mock_request: - - mock_request.post('https://test.token.endpoint', json=initial_response_json, status_code=200) + + mock_request.post("https://test.token.endpoint", json=initial_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the initial access token is set correctly - assert mock_authenticator.access_token == initial_response_json['access_token'] + assert mock_authenticator.access_token == initial_response_json["access_token"] time.sleep(2) - mock_request.post('https://test.token.endpoint', json=refresh_response_json, status_code=200) + mock_request.post("https://test.token.endpoint", json=refresh_response_json, status_code=200) mock_authenticator._get_refresh_access_token_response() # Assert that the access token is refreshed - assert mock_authenticator.access_token == refresh_response_json['access_token'] + assert mock_authenticator.access_token == refresh_response_json["access_token"] def test_backoff_retry(mock_authenticator, caplog): - - mock_response = {'access_token': 'test_access_token', 'expires_in': 3600} + + mock_response = {"access_token": "test_access_token", "expires_in": 3600} mock_reason = "Too Many Requests" - + with requests_mock.Mocker() as mock_request: - mock_request.post('https://test.token.endpoint', json=mock_response, status_code=429, reason=mock_reason) + mock_request.post("https://test.token.endpoint", json=mock_response, status_code=429, reason=mock_reason) with caplog.at_level(logging.INFO): try: mock_authenticator._get_refresh_access_token_response() @@ -67,6 +69,7 @@ def test_backoff_retry(mock_authenticator, caplog): else: pytest.fail("Expected DefaultBackoffException to be raised") + @pytest.fixture def authenticator_parameters(): return { @@ -75,14 +78,12 @@ def authenticator_parameters(): "config": {}, "parameters": {}, "token_refresh_endpoint": "https://test.token.endpoint", - "grant_type": "test_grant_type" + "grant_type": "test_grant_type", } + def test_get_headers(authenticator_parameters): expected_basic_auth = "Basic dGVzdF9jbGllbnRfaWQ6dGVzdF9jbGllbnRfc2VjcmV0" authenticator = PayPalOauth2Authenticator(**authenticator_parameters) headers = authenticator.get_headers() assert headers == {"Authorization": expected_basic_auth} - - - diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py index 06dd08dc74a6d..bd2f9c89836a8 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py @@ -11,21 +11,21 @@ @pytest.fixture(name="config") def config_fixture(): - #From File test + # From File test # with open('../secrets/config.json') as f: # return json.load(f) - #Mock test + # Mock test return { - "client_id": "your_client_id", - "client_secret": "your_client_secret", - "start_date": "2024-01-30T00:00:00Z", - "end_date": "2024-02-01T00:00:00Z", - "dispute_start_date": "2024-02-01T00:00:00.000Z", - "dispute_end_date": "2024-02-05T23:59:00.000Z", - "buyer_username": "Your Buyer email", - "buyer_password": "Your Buyer Password", - "payer_id": "ypur ACCOUNT ID", - "is_sandbox": True + "client_id": "your_client_id", + "client_secret": "your_client_secret", + "start_date": "2024-01-30T00:00:00Z", + "end_date": "2024-02-01T00:00:00Z", + "dispute_start_date": "2024-02-01T00:00:00.000Z", + "dispute_end_date": "2024-02-05T23:59:00.000Z", + "buyer_username": "Your Buyer email", + "buyer_password": "Your Buyer Password", + "payer_id": "ypur ACCOUNT ID", + "is_sandbox": True, } @@ -33,21 +33,24 @@ def config_fixture(): def source_fixture(): return SourcePaypalTransaction() + def validate_date_format(date_str, format): try: datetime.strptime(date_str, format) return True except ValueError: return False - + + def test_date_formats_in_config(config): start_date_format = "%Y-%m-%dT%H:%M:%SZ" dispute_date_format = "%Y-%m-%dT%H:%M:%S.%fZ" - assert validate_date_format(config['start_date'], start_date_format), "Start date format is incorrect" - assert validate_date_format(config['end_date'], start_date_format), "End date format is incorrect" - assert validate_date_format(config['dispute_start_date'], dispute_date_format), "Dispute start date format is incorrect" - assert validate_date_format(config['dispute_end_date'], dispute_date_format), "Dispute end date format is incorrect" + assert validate_date_format(config["start_date"], start_date_format), "Start date format is incorrect" + assert validate_date_format(config["end_date"], start_date_format), "End date format is incorrect" + assert validate_date_format(config["dispute_start_date"], dispute_date_format), "Dispute start date format is incorrect" + assert validate_date_format(config["dispute_end_date"], dispute_date_format), "Dispute end date format is incorrect" + @pytest.fixture(name="logger_mock") def logger_mock_fixture(): - return patch("source_paypal_transactions.source.AirbyteLogger") \ No newline at end of file + return patch("source_paypal_transactions.source.AirbyteLogger") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py index 958db41262da2..7ec7286fc7310 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_cursor.py @@ -27,23 +27,23 @@ class CursorPaginationStrategy(PaginationStrategy): stop_condition (Optional[InterpolatedBoolean]): template string evaluating when to stop paginating decoder (Decoder): decoder to decode the response """ + cursor_value: Union[InterpolatedString, str] config: Config parameters: Mapping[str, Any] page_size: Optional[int] = None stop_condition: Optional[Union[InterpolatedBoolean, str]] = None decoder: Decoder = field(default_factory=JsonDecoder) - + def __post_init__(self): if isinstance(self.cursor_value, str): self.cursor_value = InterpolatedString.create(self.cursor_value, parameters=self.parameters) if isinstance(self.stop_condition, str): self.stop_condition = InterpolatedBoolean(condition=self.stop_condition, parameters=self.parameters) - + @property def initial_token(self) -> Optional[Any]: return None - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) @@ -51,53 +51,45 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin headers["link"] = response.links print("STOP CONDITION", self.stop_condition) - + if self.stop_condition: should_stop = self.stop_condition.eval(self.config, response=decoded_response, headers=headers, last_records=last_records) if should_stop: print("Stopping...") return None - + # Update cursor_value with the next_id from the response self.cursor_value = InterpolatedString.create(decoded_response.get("next_id"), parameters=self.parameters) token = self.cursor_value.eval(config=self.config, last_records=last_records, response=decoded_response, headers=headers) print("TOKEN", token) return token if token else None - + def reset(self): pass - + def get_page_size(self) -> Optional[int]: return self.page_size @pytest.fixture def mock_responses(): - return [ - "token_page_init.json", - "token_PAY-0L38757939422510JMW5ZJVA.json", - "token_PAYID-MW5XXZY5YL87592N34454913.json" - ] + return ["token_page_init.json", "token_PAY-0L38757939422510JMW5ZJVA.json", "token_PAYID-MW5XXZY5YL87592N34454913.json"] + @pytest.fixture -def cursor_pagination_strategy(mock_responses, stop_condition = None): +def cursor_pagination_strategy(mock_responses, stop_condition=None): parameters = {} decoder = JsonDecoder(parameters=parameters) cursor_value = "start_id" # Initialize with a default value - + for response_file in mock_responses: if cursor_value == "start_id": cursor_value = load_mock_data(response_file).get("next_id") else: break # Stop after getting the next_id from the first response - + return CursorPaginationStrategy( - cursor_value=cursor_value, - config={}, - parameters=parameters, - page_size=3, - stop_condition=stop_condition, - decoder=decoder + cursor_value=cursor_value, config={}, parameters=parameters, page_size=3, stop_condition=stop_condition, decoder=decoder ) @@ -105,6 +97,7 @@ def load_mock_data(filename): with open(os.path.join("./unit_tests/test_files", filename), "r") as file: return json.load(file) + def test_cursor_pagination(cursor_pagination_strategy, mock_responses): with requests_mock.Mocker() as m: base_url = "http://example.com/api/resource" @@ -126,21 +119,21 @@ def test_cursor_pagination(cursor_pagination_strategy, mock_responses): if i < len(mock_responses) - 1: next_id = load_mock_data(response_file)["next_id"] print("FOUND NEXT ID:", next_id) - + else: next_id = None - cursor_pagination_strategy(mock_responses, stop_condition = True) + cursor_pagination_strategy(mock_responses, stop_condition=True) # Make API call and process response response = requests.get(url) print("GET RESPONSE:", response) assert response.status_code == 200 - + decoded_response = response.json() last_records = decoded_response["payments"] next_id = cursor_pagination_strategy.next_page_token(response, last_records) print("NEXT ID:", next_id) - + # Verify the pagination stopped assert next_id is None print("No more pages") diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py index 05b98d04f90a1..95bdfccf91200 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/pagination_increment.py @@ -29,24 +29,25 @@ def reset(self): def get_page_size(self): return self.page_size + @pytest.fixture def mock_pagination_strategy(): return MockPaginationStrategy(page_size=500) + @pytest.fixture def paginator(): pagination_strategy = MockPaginationStrategy(page_size=3) return DefaultPaginator( - pagination_strategy=pagination_strategy, - config={}, - url_base="http://example.com/v1/reporting/transactions", - parameters={} + pagination_strategy=pagination_strategy, config={}, url_base="http://example.com/v1/reporting/transactions", parameters={} ) - + + def load_mock_data(page): with open(f"./unit_tests/test_files/page_{page}.json", "r") as file: return file.read() + # Test to verify pagination logic transitions from page 1 to page 2 def test_pagination_logic(paginator): page_1_data = load_mock_data(1) @@ -54,7 +55,7 @@ def test_pagination_logic(paginator): paginator_url_1 = f"{paginator.url_base.string}?page=1&page_size={paginator.pagination_strategy.get_page_size}" paginator_url_2 = f"{paginator.url_base.string}?page=2&page_size={paginator.pagination_strategy.get_page_size}" - + with requests_mock.Mocker() as m: m.get(paginator_url_1, text=page_1_data, status_code=200) m.get(paginator_url_2, text=page_2_data, status_code=200) @@ -64,16 +65,14 @@ def test_pagination_logic(paginator): response_page_2 = requests.get(paginator_url_2) response_page_2._content = str.encode(page_2_data) - # Simulate getting the next page token from page 1's response next_page_token_page_1 = paginator.next_page_token(response_page_1, []) print("NEXT PAGE TOKEN", next_page_token_page_1) # Assert that the next page token indicates moving to page 2 - assert next_page_token_page_1['next_page_token'] == 2, "Failed to transition from page 1 to page 2" + assert next_page_token_page_1["next_page_token"] == 2, "Failed to transition from page 1 to page 2" - # Check that the correct page size is used in requests and that we have the right number of pages - assert len(m.request_history) == 2 - assert "page_size=3" in m.request_history[0].url - assert "page_size=3" in m.request_history[1].url \ No newline at end of file + assert len(m.request_history) == 2 + assert "page_size=3" in m.request_history[0].url + assert "page_size=3" in m.request_history[1].url diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py index 86cc6f23f8f4d..384437cc0fe55 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_auth.py @@ -17,6 +17,7 @@ resp = Response() + class TestPinterestOauthAuthenticator: """ Test class for custom PinterestOauthAuthenticator, derived from the CDK's Oauth2Authenticator class. @@ -69,7 +70,9 @@ def test_refresh_access_token_invalid_or_expired(self, mocker, oauth): mocker.patch.object(resp, "status_code", 400) mocker.patch.object(oauth, "_wrap_refresh_token_exception", return_value=True) - with pytest.raises(AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings."): + with pytest.raises( + AirbyteTracedException, match="Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." + ): oauth.refresh_access_token() diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py index 3dd38604a86eb..c703264a779af 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py @@ -135,7 +135,8 @@ def test_semi_incremental_read(requests_mock, test_config, start_date, stream_st stream.state = stream_state actual_records = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.incremental) for record in stream.read_records(sync_mode=SyncMode.incremental, stream_slice=stream_slice) ] assert actual_records == expected_records diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index df5c903ee3475..1479b3f605594 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -24,7 +24,7 @@ ) from source_pinterest.utils import get_analytics_columns -os.environ["REQUEST_CACHE_PATH"] = '/tmp' +os.environ["REQUEST_CACHE_PATH"] = "/tmp" def test_request_body_json(analytics_report_stream, date_range): @@ -79,18 +79,20 @@ def test_streams(test_config): def test_custom_streams(test_config): config = copy.deepcopy(test_config) - config['custom_reports'] = [{ - "name": "vadim_report", - "level": "AD_GROUP", - "granularity": "MONTH", - "click_window_days": 30, - "engagement_window_days": 30, - "view_window_days": 30, - "conversion_report_time": "TIME_OF_CONVERSION", - "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], - "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], - "start_date": "2023-01-08" - }] + config["custom_reports"] = [ + { + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08", + } + ] source = SourcePinterest() streams = source.streams(config) expected_streams_number = 33 @@ -100,18 +102,18 @@ def test_custom_streams(test_config): @pytest.mark.parametrize( ("report_name", "expected_level"), ( - [CampaignAnalyticsReport, 'CAMPAIGN'], - [CampaignTargetingReport, 'CAMPAIGN_TARGETING'], - [AdvertiserReport, 'ADVERTISER'], - [AdvertiserTargetingReport, 'ADVERTISER_TARGETING'], - [AdGroupReport, 'AD_GROUP'], - [AdGroupTargetingReport, 'AD_GROUP_TARGETING'], - [PinPromotionReport, 'PIN_PROMOTION'], - [PinPromotionTargetingReport, 'PIN_PROMOTION_TARGETING'], - [ProductGroupReport, 'PRODUCT_GROUP'], - [ProductGroupTargetingReport, 'PRODUCT_GROUP_TARGETING'], - [ProductItemReport, 'PRODUCT_ITEM'], - [KeywordReport, 'KEYWORD'] + [CampaignAnalyticsReport, "CAMPAIGN"], + [CampaignTargetingReport, "CAMPAIGN_TARGETING"], + [AdvertiserReport, "ADVERTISER"], + [AdvertiserTargetingReport, "ADVERTISER_TARGETING"], + [AdGroupReport, "AD_GROUP"], + [AdGroupTargetingReport, "AD_GROUP_TARGETING"], + [PinPromotionReport, "PIN_PROMOTION"], + [PinPromotionTargetingReport, "PIN_PROMOTION_TARGETING"], + [ProductGroupReport, "PRODUCT_GROUP"], + [ProductGroupTargetingReport, "PRODUCT_GROUP_TARGETING"], + [ProductItemReport, "PRODUCT_ITEM"], + [KeywordReport, "KEYWORD"], ), ) def test_level(test_config, report_name, expected_level): diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 6012f9d1b2118..9f46bef6f4064 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -30,8 +30,7 @@ def test_check_connection_expired_token(requests_mock, test_config): logger_mock = MagicMock() assert source.check_connection(logger_mock, test_config) == ( False, - "Unable to connect to stream boards - 401 Client Error: None " - "for url: https://api.pinterest.com/v5/oauth/token", + "Unable to connect to stream boards - 401 Client Error: None " "for url: https://api.pinterest.com/v5/oauth/token", ) diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 80f5d4aa37a75..d9f3949185d76 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -80,7 +80,8 @@ def test_parse_response_with_sensitive_data(requests_mock, test_config): json={"items": [{"id": "CatalogsFeeds1", "credentials": {"password": "bla"}}]}, ) actual_response = [ - dict(record) for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) + dict(record) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh) for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) ] assert actual_response == [{"id": "CatalogsFeeds1"}] @@ -122,7 +123,9 @@ def test_response_action(requests_mock, patch_base_class, http_status, expected_ ), ) @patch("time.sleep", return_value=None) -def test_declarative_stream_response_action_on_max_rate_limit_error(mock_sleep, requests_mock, test_response, status_code, expected_response_action): +def test_declarative_stream_response_action_on_max_rate_limit_error( + mock_sleep, requests_mock, test_response, status_code, expected_response_action +): response_mock = create_requests_response(requests_mock, status_code, {}) error_handler = PinterestErrorHandler(logger=MagicMock(), stream_name="any_stream_name") assert error_handler.interpret_response(response_mock).response_action == expected_response_action diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py index 56afb08757069..580bb6508f0c7 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/hook.py @@ -20,12 +20,13 @@ abnormal_state_write_file = "/connector/integration_tests/temp/abnormal_state_copy.json" abnormal_state_file = "/connector/integration_tests/abnormal_state_template.json" -secret_config_file = '/connector/secrets/config.json' -secret_active_config_file = '/connector/integration_tests/temp/config_active.json' -secret_config_cdc_file = '/connector/secrets/config_cdc.json' -secret_active_config_cdc_file = '/connector/integration_tests/temp/config_cdc_active.json' +secret_config_file = "/connector/secrets/config.json" +secret_active_config_file = "/connector/integration_tests/temp/config_active.json" +secret_config_cdc_file = "/connector/secrets/config_cdc.json" +secret_active_config_cdc_file = "/connector/integration_tests/temp/config_cdc_active.json" + +la_timezone = pytz.timezone("America/Los_Angeles") -la_timezone = pytz.timezone('America/Los_Angeles') def connect_to_db() -> extensions.connection: with open(secret_config_file) as f: @@ -33,11 +34,7 @@ def connect_to_db() -> extensions.connection: try: conn: extensions.connection = psycopg2.connect( - dbname=secret["database"], - user=secret["username"], - password=secret["password"], - host=secret["host"], - port=secret["port"] + dbname=secret["database"], user=secret["username"], password=secret["password"], host=secret["host"], port=secret["port"] ) print("Connected to the database successfully") return conn @@ -45,13 +42,16 @@ def connect_to_db() -> extensions.connection: print(f"Error connecting to the database: {error}") sys.exit(1) + def insert_records(conn: extensions.connection, schema_name: str, table_name: str, records: List[Tuple[str, str]]) -> None: try: cursor = conn.cursor() - insert_query = sql.SQL(""" + insert_query = sql.SQL( + """ INSERT INTO {}.{} (id, name) VALUES (%s, %s) ON CONFLICT DO NOTHING - """).format(sql.Identifier(schema_name), sql.Identifier(table_name)) + """ + ).format(sql.Identifier(schema_name), sql.Identifier(table_name)) for record in records: cursor.execute(insert_query, record) @@ -64,6 +64,7 @@ def insert_records(conn: extensions.connection, schema_name: str, table_name: st finally: cursor.close() + def create_schema(conn: extensions.connection, schema_name: str) -> None: try: cursor = conn.cursor() @@ -77,24 +78,25 @@ def create_schema(conn: extensions.connection, schema_name: str) -> None: finally: cursor.close() + def write_supporting_file(schema_name: str) -> None: print(f"writing schema name to files: {schema_name}") Path("/connector/integration_tests/temp").mkdir(parents=False, exist_ok=True) with open(catalog_write_file, "w") as file: - with open(catalog_source_file, 'r') as source_file: + with open(catalog_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(catalog_incremental_write_file, "w") as file: - with open(catalog_incremental_source_file, 'r') as source_file: + with open(catalog_incremental_source_file, "r") as source_file: file.write(source_file.read() % schema_name) with open(abnormal_state_write_file, "w") as file: - with open(abnormal_state_file, 'r') as source_file: + with open(abnormal_state_file, "r") as source_file: file.write(source_file.read() % (schema_name, schema_name)) with open(secret_config_file) as base_config: secret = json.load(base_config) secret["schemas"] = [schema_name] - with open(secret_active_config_file, 'w') as f: + with open(secret_active_config_file, "w") as f: json.dump(secret, f) with open(secret_config_cdc_file) as base_config: @@ -104,18 +106,21 @@ def write_supporting_file(schema_name: str) -> None: secret["replication_method"]["publication"] = schema_name secret["ssl_mode"] = {} secret["ssl_mode"]["mode"] = "require" - with open(secret_active_config_cdc_file, 'w') as f: + with open(secret_active_config_cdc_file, "w") as f: json.dump(secret, f) + def create_table(conn: extensions.connection, schema_name: str, table_name: str) -> None: try: cursor = conn.cursor() - create_table_query = sql.SQL(""" + create_table_query = sql.SQL( + """ CREATE TABLE IF NOT EXISTS {}.{} ( id VARCHAR(100) PRIMARY KEY, name VARCHAR(255) NOT NULL ) - """).format(sql.Identifier(schema_name), sql.Identifier(table_name)) + """ + ).format(sql.Identifier(schema_name), sql.Identifier(table_name)) cursor.execute(create_table_query) conn.commit() @@ -126,40 +131,37 @@ def create_table(conn: extensions.connection, schema_name: str, table_name: str) finally: cursor.close() + def generate_schema_date_with_suffix() -> str: current_date = datetime.datetime.now(la_timezone).strftime("%Y%m%d") - suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8)) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=8)) return f"{current_date}_{suffix}" + def prepare() -> None: schema_name = generate_schema_date_with_suffix() print(f"schema_name: {schema_name}") with open("./generated_schema.txt", "w") as f: f.write(schema_name) + def cdc_insert(): schema_name = load_schema_name_from_catalog() - new_records = [ - ('4', 'four'), - ('5', 'five') - ] + new_records = [("4", "four"), ("5", "five")] connection = connect_to_db() - table_name = 'id_and_name_cat' + table_name = "id_and_name_cat" if connection: insert_records(connection, schema_name, table_name, new_records) connection.close() + def setup(with_cdc=False): schema_name = load_schema_name_from_catalog() write_supporting_file(schema_name) table_name = "id_and_name_cat" # Define the records to be inserted - records = [ - ('1', 'one'), - ('2', 'two'), - ('3', 'three') - ] + records = [("1", "one"), ("2", "two"), ("3", "three")] # Connect to the database connection = connect_to_db() @@ -167,7 +169,7 @@ def setup(with_cdc=False): # Create the schema create_schema(connection, schema_name) create_table(connection, schema_name, table_name) - if (with_cdc): + if with_cdc: setup_cdc(connection, replication_slot_and_publication_name=schema_name) # Insert the records insert_records(connection, schema_name, table_name, records) @@ -175,6 +177,7 @@ def setup(with_cdc=False): # Close the connection connection.close() + def replication_slot_existed(connection, replication_slot_name): cursor = connection.cursor() cursor.execute("SELECT slot_name FROM pg_replication_slots;") @@ -185,22 +188,31 @@ def replication_slot_existed(connection, replication_slot_name): return True return False + def setup_cdc(connection, replication_slot_and_publication_name): cursor = connection.cursor() if replication_slot_existed(connection, replication_slot_and_publication_name): return - create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format(sql.Literal(replication_slot_and_publication_name)) + create_logical_replication_query = sql.SQL("SELECT pg_create_logical_replication_slot({}, 'pgoutput')").format( + sql.Literal(replication_slot_and_publication_name) + ) cursor.execute(create_logical_replication_query) - alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format(sql.Identifier(replication_slot_and_publication_name)) + alter_table_replica_query = sql.SQL("ALTER TABLE {}.id_and_name_cat REPLICA IDENTITY DEFAULT").format( + sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(alter_table_replica_query) - create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format(sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name)) + create_publication_query = sql.SQL("CREATE PUBLICATION {} FOR TABLE {}.id_and_name_cat").format( + sql.Identifier(replication_slot_and_publication_name), sql.Identifier(replication_slot_and_publication_name) + ) cursor.execute(create_publication_query) connection.commit() + def load_schema_name_from_catalog(): with open("./generated_schema.txt", "r") as f: return f.read() + def delete_cdc_with_prefix(conn, prefix): try: # Connect to the PostgreSQL database @@ -224,17 +236,20 @@ def delete_cdc_with_prefix(conn, prefix): if cursor: cursor.close() + def delete_schemas_with_prefix(conn, date_prefix): try: # Connect to the PostgreSQL database cursor = conn.cursor() # Query to find all schemas that start with the specified date prefix - query = sql.SQL(""" + query = sql.SQL( + """ SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE %s; - """) + """ + ) cursor.execute(query, (f"{date_prefix}%",)) schemas = cursor.fetchall() @@ -252,14 +267,16 @@ def delete_schemas_with_prefix(conn, date_prefix): finally: cursor.close() + def teardown() -> None: conn = connect_to_db() today = datetime.datetime.now(la_timezone) yesterday = today - timedelta(days=1) - formatted_yesterday = yesterday.strftime('%Y%m%d') + formatted_yesterday = yesterday.strftime("%Y%m%d") delete_schemas_with_prefix(conn, formatted_yesterday) delete_cdc_with_prefix(conn, formatted_yesterday) + def final_teardown() -> None: conn = connect_to_db() schema_name = load_schema_name_from_catalog() @@ -267,6 +284,7 @@ def final_teardown() -> None: delete_schemas_with_prefix(conn, schema_name) delete_cdc_with_prefix(conn, schema_name) + if __name__ == "__main__": command = sys.argv[1] if command == "setup": diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index d6a06768d52f6..cebf1c6765490 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -47,7 +47,7 @@ def with_access_token(self, access_token: str) -> RequestBuilder: def with_old_api_version(self, api_version: str) -> RequestBuilder: self._headers["X-Recharge-Version"] = api_version return self - + def with_created_min(self, value: str) -> RequestBuilder: self._query_params["created_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) return self diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index af3ba6d425b82..633a256469ce0 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -88,6 +88,7 @@ def test_should_retry(self, config, http_status, headers, expected_action) -> No error_resolution = stream.get_error_handler().interpret_response(response) error_resolution.response_action == expected_action + class TestFullRefreshStreams: def generate_records(self, stream_name, count) -> Union[Mapping[str, List[Mapping[str, Any]]], Mapping[str, Any]]: if not stream_name: diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py index 26647e4f62ad9..d6356394f1cd3 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/test_acceptance.py @@ -70,6 +70,7 @@ def expect_exception(self) -> bool: def instance_name(self) -> str: return self.config_path.stem + def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: all_tests_config = yaml.safe_load(ACCEPTANCE_TEST_CONFIG_PATH.read_text()) return [ @@ -78,6 +79,7 @@ def get_acceptance_tests(category: str) -> list[AcceptanceTestInstance]: if "iam_role" not in test["config_path"] ] + # TODO: Convert to a CDK class for better reuse and portability. # class TestSourceAcceptanceTestSuiteBase: # """Test suite for acceptance tests.""" @@ -233,7 +235,9 @@ def test_check(instance: AcceptanceTestInstance) -> None: "check", test_instance=instance, ) - conn_status_messages: list[AirbyteMessage] = [msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS] # noqa: SLF001 # Non-public API + conn_status_messages: list[AirbyteMessage] = [ + msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS + ] # noqa: SLF001 # Non-public API assert len(conn_status_messages) == 1, "Expected exactly one CONNECTION_STATUS message. Got: \n" + "\n".join(result._messages) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py index 4d04411a66942..5db2e4264d75e 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py @@ -134,9 +134,7 @@ def _transform_file_format(cls, format_options: Union[CsvFormat, ParquetFormat, raise ValueError( "The config options you selected are no longer supported.\n" + f"advanced_options={advanced_options}" if advanced_options - else "" + f"additional_reader_options={additional_reader_options}" - if additional_reader_options - else "" + else "" + f"additional_reader_options={additional_reader_options}" if additional_reader_options else "" ) return csv_options diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 02e76e5790ea6..d952b8b91af85 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -124,7 +124,7 @@ def test_get_matching_files( except Exception as exc: raise exc - with patch.object(SourceS3StreamReader, 's3_client', new_callable=MagicMock) as mock_s3_client: + with patch.object(SourceS3StreamReader, "s3_client", new_callable=MagicMock) as mock_s3_client: _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages) files = list(reader.get_matching_files(globs, None, logger)) assert set(f.uri for f in files) == expected_uris @@ -134,27 +134,33 @@ def _setup_mock_s3_client(mock_s3_client, mocked_response, multiple_pages): responses = [] if multiple_pages and len(mocked_response) > 1: # Split the mocked_response for pagination simulation - first_half = mocked_response[:len(mocked_response) // 2] - second_half = mocked_response[len(mocked_response) // 2:] - - responses.append({ - "IsTruncated": True, - "Contents": first_half, - "KeyCount": len(first_half), - "NextContinuationToken": "token", - }) - - responses.append({ - "IsTruncated": False, - "Contents": second_half, - "KeyCount": len(second_half), - }) + first_half = mocked_response[: len(mocked_response) // 2] + second_half = mocked_response[len(mocked_response) // 2 :] + + responses.append( + { + "IsTruncated": True, + "Contents": first_half, + "KeyCount": len(first_half), + "NextContinuationToken": "token", + } + ) + + responses.append( + { + "IsTruncated": False, + "Contents": second_half, + "KeyCount": len(second_half), + } + ) else: - responses.append({ - "IsTruncated": False, - "Contents": mocked_response, - "KeyCount": len(mocked_response), - }) + responses.append( + { + "IsTruncated": False, + "Contents": mocked_response, + "KeyCount": len(mocked_response), + } + ) def list_objects_v2_side_effect(Bucket, Prefix=None, ContinuationToken=None, **kwargs): if ContinuationToken == "token": @@ -252,7 +258,13 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): mock_s3_client_instance.download_file.return_value = None reader = SourceS3StreamReader() - reader.config = Config(bucket="test", aws_access_key_id="test", aws_secret_access_key="test", streams=[], delivery_method= { "delivery_type": "use_file_transfer" }) + reader.config = Config( + bucket="test", + aws_access_key_id="test", + aws_secret_access_key="test", + streams=[], + delivery_method={"delivery_type": "use_file_transfer"}, + ) try: reader.config = Config( bucket="test", @@ -260,14 +272,14 @@ def test_get_file(mock_boto_client, s3_reader_file_size_mock): aws_secret_access_key="test", streams=[], endpoint=None, - delivery_method={"delivery_type": "use_file_transfer"} + delivery_method={"delivery_type": "use_file_transfer"}, ) except Exception as exc: raise exc test_file_path = "directory/file.txt" result = reader.get_file(RemoteFile(uri="", last_modified=datetime.now()), test_file_path, logger) - assert result == {'bytes': 100, 'file_relative_path': ANY, 'file_url': ANY} + assert result == {"bytes": 100, "file_relative_path": ANY, "file_url": ANY} assert result["file_url"].endswith(test_file_path) @@ -317,27 +329,20 @@ def test_get_iam_s3_client(boto3_client_mock): # Assertions to validate the s3 client assert s3_client is not None + @pytest.mark.parametrize( "start_date, last_modified_date, expected_result", ( # True when file is new or modified after given start_date - ( - datetime.now() - timedelta(days=180), - datetime.now(), - True - ), + (datetime.now() - timedelta(days=180), datetime.now(), True), ( datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), datetime.strptime("2024-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), - True + True, ), # False when file is older than given start_date - ( - datetime.now(), - datetime.now() - timedelta(days=180), - False - ) - ) + (datetime.now(), datetime.now() - timedelta(days=180), False), + ), ) def test_filter_file_by_start_date(start_date: datetime, last_modified_date: datetime, expected_result: bool) -> None: reader = SourceS3StreamReader() @@ -347,7 +352,7 @@ def test_filter_file_by_start_date(start_date: datetime, last_modified_date: dat aws_access_key_id="test", aws_secret_access_key="test", streams=[], - start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ") + start_date=start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), ) assert expected_result == reader.is_modified_after_start_date(last_modified_date) diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py index 1742ef72a9236..e14b1484d5e30 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py @@ -51,7 +51,11 @@ def stream_name(): @pytest.fixture(scope="module") def stream(input_sandbox_config, stream_name, sf): - return SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE).generate_streams(input_sandbox_config, {stream_name: None}, sf)[0]._legacy_stream + return ( + SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) + .generate_streams(input_sandbox_config, {stream_name: None}, sf)[0] + ._legacy_stream + ) def _encode_content(text): diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py index d03de9ed39e8f..26538e4c4705d 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/state_migration.py @@ -2616,7 +2616,7 @@ for stream in json.loads(x): stream["stream_descriptor"] = stream.pop("streamDescriptor") stream["stream_state"] = stream.pop("streamState") - y = { + y = { "type": "STREAM", "stream": stream, } diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py index 2fa5db0588052..8f7c32a7bc392 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py @@ -96,9 +96,11 @@ def interpret_response(self, response: Optional[Union[requests.Response, Excepti return ErrorResolution( ResponseAction.FAIL, FailureType.config_error, - _AUTHENTICATION_ERROR_MESSAGE_MAPPING.get(error_message) - if error_message in _AUTHENTICATION_ERROR_MESSAGE_MAPPING - else f"An error occurred: {response.content.decode()}", + ( + _AUTHENTICATION_ERROR_MESSAGE_MAPPING.get(error_message) + if error_message in _AUTHENTICATION_ERROR_MESSAGE_MAPPING + else f"An error occurred: {response.content.decode()}" + ), ) if self._is_bulk_job_creation(response) and response.status_code in [codes.FORBIDDEN, codes.BAD_REQUEST]: diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 1a59386a1b952..9e07453c92381 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -100,9 +100,7 @@ def test_stream_slice_step_validation(stream_slice_step: str, expected_error_mes ), ], ) -def test_login_authentication_error_handler( - stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg -): +def test_login_authentication_error_handler(stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg): source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG, _ANY_STATE) logger = logging.getLogger("airbyte") requests_mock.register_uri( @@ -557,13 +555,21 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True) job_id_1 = "fake_job_1" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}", [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_1}", + [{"json": JobInfoResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_1}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_1}/results", text="Field1,LastModifiedDate,ID\ntest,2023-01-15,1") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_1}") job_id_2 = "fake_job_2" - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_2}", [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_2}", + [{"json": JobInfoResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_2}") requests_mock.register_uri( "GET", _bulk_stream_path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22" @@ -574,7 +580,11 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api queries_history = requests_mock.register_uri( "POST", _bulk_stream_path(), [{"json": {"id": job_id_1}}, {"json": {"id": job_id_2}}, {"json": {"id": job_id_3}}] ) - requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}", [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}]) + requests_mock.register_uri( + "GET", + _bulk_stream_path() + f"/{job_id_3}", + [{"json": JobInfoResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}], + ) requests_mock.register_uri("DELETE", _bulk_stream_path() + f"/{job_id_3}") requests_mock.register_uri("GET", _bulk_stream_path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3") requests_mock.register_uri("PATCH", _bulk_stream_path() + f"/{job_id_3}") @@ -586,18 +596,24 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api # assert request params: has requests might not be performed in a specific order because of concurrent CDK, we match on any request all_requests = {request.text for request in queries_history.request_history} - assert any([ - "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" - in request for request in all_requests - ]) - assert any([ - "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" - in request for request in all_requests - ]) + assert any( + [ + "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" in request + for request in all_requests + ] + ) + assert any( + [ + "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" in request + for request in all_requests + ] + ) # as the execution is concurrent, we can only assert the last state message here last_actual_state = [item.state.stream.stream_state for item in result if item.type == Type.STATE][-1] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py index a5156457d1f25..dbb5b64be189b 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py @@ -25,7 +25,9 @@ _CLIENT_SECRET = "a_client_secret" _CURSOR_FIELD = "SystemModstamp" _INCREMENTAL_FIELDS = [_A_FIELD_NAME, _CURSOR_FIELD] -_INCREMENTAL_SCHEMA_BUILDER = SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") # re-using same fields as _INCREMENTAL_FIELDS +_INCREMENTAL_SCHEMA_BUILDER = ( + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime") +) # re-using same fields as _INCREMENTAL_FIELDS _INSTANCE_URL = "https://instance.salesforce.com" _JOB_ID = "a-job-id" _ANOTHER_JOB_ID = "another-job-id" @@ -60,17 +62,16 @@ def _calculate_start_time(start_time: datetime) -> datetime: def _build_job_creation_request(query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps({"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"}), + ) def _make_sliced_job_request(lower_boundary: datetime, upper_boundary: datetime, fields: List[str]) -> HttpRequest: - return _build_job_creation_request(f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + return _build_job_creation_request( + f"SELECT {', '.join(fields)} FROM a_stream_name WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(fields: List[str]) -> HttpRequest: @@ -168,7 +169,9 @@ def test_given_type_when_read_then_field_is_casted_with_right_type(self) -> None @freezegun.freeze_time(_NOW.isoformat()) def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -192,7 +195,9 @@ def test_given_no_data_provided_when_read_then_field_is_none(self) -> None: @freezegun.freeze_time(_NOW.isoformat()) def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -220,7 +225,9 @@ def test_given_csv_unix_dialect_provided_when_read_then_parse_csv_properly(self) @freezegun.freeze_time(_NOW.isoformat()) def test_given_specific_encoding_when_read_then_parse_csv_properly(self) -> None: - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME)) + given_stream( + self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_ANOTHER_FIELD_NAME) + ) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME, _ANOTHER_FIELD_NAME]), JobCreateResponseBuilder().with_id(_JOB_ID).build(), @@ -338,7 +345,17 @@ def test_given_non_transient_error_on_job_creation_when_read_then_fail_sync(self given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) self._http_mocker.post( _make_full_job_request([_A_FIELD_NAME]), - HttpResponse(json.dumps([{"errorCode": "API_ERROR", "message": "Implementation restriction... "}]), 400), + HttpResponse( + json.dumps( + [ + { + "errorCode": "API_ERROR", + "message": "Implementation restriction... ", + } + ] + ), + 400, + ), ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -526,11 +543,21 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) self._config.start_date(start_date).stream_slice_step("P7D") given_stream(self._http_mocker, _BASE_URL, _STREAM_WITH_PARENT_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME)) - self._create_sliced_job_with_records(start_date, first_upper_boundary, _PARENT_STREAM_NAME, "first_parent_slice_job_id", [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}]) - self._create_sliced_job_with_records(first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}]) + self._create_sliced_job_with_records( + start_date, + first_upper_boundary, + _PARENT_STREAM_NAME, + "first_parent_slice_job_id", + [{"Id": "parent1", "SystemModstamp": "any"}, {"Id": "parent2", "SystemModstamp": "any"}], + ) + self._create_sliced_job_with_records( + first_upper_boundary, _NOW, _PARENT_STREAM_NAME, "second_parent_slice_job_id", [{"Id": "parent3", "SystemModstamp": "any"}] + ) self._http_mocker.post( - self._build_job_creation_request(f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')"), + self._build_job_creation_request( + f"SELECT {', '.join([_A_FIELD_NAME])} FROM {_STREAM_WITH_PARENT_NAME} WHERE ContentDocumentId IN ('parent1', 'parent2', 'parent3')" + ), JobCreateResponseBuilder().with_id(_JOB_ID).build(), ) self._http_mocker.get( @@ -547,10 +574,16 @@ def test_given_parent_stream_when_read_then_return_record_for_all_children(self) assert len(output.records) == 1 - def _create_sliced_job(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int) -> None: - self._create_sliced_job_with_records(lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count)) + def _create_sliced_job( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str], job_id: str, record_count: int + ) -> None: + self._create_sliced_job_with_records( + lower_boundary, upper_boundary, stream_name, job_id, self._generate_random_records(fields, record_count) + ) - def _create_sliced_job_with_records(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]]) -> None: + def _create_sliced_job_with_records( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, job_id: str, records: List[Dict[str, str]] + ) -> None: self._http_mocker.post( self._make_sliced_job_request(lower_boundary, upper_boundary, stream_name, list(records[0].keys())), JobCreateResponseBuilder().with_id(job_id).build(), @@ -581,8 +614,12 @@ def _create_csv(self, headers: List[str], data: List[Dict[str, str]], dialect: s writer.writerow(line) return csvfile.getvalue() - def _make_sliced_job_request(self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str]) -> HttpRequest: - return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}") + def _make_sliced_job_request( + self, lower_boundary: datetime, upper_boundary: datetime, stream_name: str, fields: List[str] + ) -> HttpRequest: + return self._build_job_creation_request( + f"SELECT {', '.join(fields)} FROM {stream_name} WHERE SystemModstamp >= {lower_boundary.isoformat(timespec='milliseconds')} AND SystemModstamp < {upper_boundary.isoformat(timespec='milliseconds')}" + ) def _make_full_job_request(self, fields: List[str], stream_name: str = _STREAM_NAME) -> HttpRequest: return self._build_job_creation_request(f"SELECT {', '.join(fields)} FROM {stream_name}") @@ -601,14 +638,13 @@ def _generate_csv(self, records: List[Dict[str, str]]) -> str: for record in records: csv_entry.append(",".join([record[key] for key in keys])) - entries = '\n'.join(csv_entry) + entries = "\n".join(csv_entry) return f"{','.join(keys)}\n{entries}" def _build_job_creation_request(self, query: str) -> HttpRequest: - return HttpRequest(f"{_BASE_URL}/jobs/query", body=json.dumps({ - "operation": "queryAll", - "query": query, - "contentType": "CSV", - "columnDelimiter": "COMMA", - "lineEnding": "LF" - })) + return HttpRequest( + f"{_BASE_URL}/jobs/query", + body=json.dumps( + {"operation": "queryAll", "query": query, "contentType": "CSV", "columnDelimiter": "COMMA", "lineEnding": "LF"} + ), + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py index 85805d29cb749..9b241ba0aec74 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_rest_stream.py @@ -31,7 +31,7 @@ def create_http_request(stream_name: str, field_names: List[str], access_token: Optional[str] = None) -> HttpRequest: return HttpRequest( f"{_BASE_URL}/queryAll?q=SELECT+{','.join(field_names)}+FROM+{stream_name}+", - headers={"Authorization": f"Bearer {access_token}"} if access_token else None + headers={"Authorization": f"Bearer {access_token}"} if access_token else None, ) @@ -40,7 +40,10 @@ def create_http_response(field_names: List[str], record_count: int = 1) -> HttpR This method does not handle field types for now which may cause some test failures on change if we start considering using some fields for calculation. One example of that would be cursor field parsing to datetime. """ - records = [{field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} for i in range(record_count)] + records = [ + {field: "2021-01-18T21:18:20.000Z" if field in {"SystemModstamp"} else f"{field}_value" for field in field_names} + for i in range(record_count) + ] return HttpResponse(json.dumps({"records": records})) @@ -77,7 +80,7 @@ def test_given_error_on_fetch_chunk_of_properties_when_read_then_retry(self, htt [ HttpResponse("", status_code=406), create_http_response([_A_FIELD_NAME], record_count=1), - ] + ], ) output = read(_STREAM_NAME, SyncMode.full_refresh, self._config) @@ -94,7 +97,12 @@ def setUp(self) -> None: self._http_mocker.__enter__() given_authentication(self._http_mocker, _CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN, _INSTANCE_URL) - given_stream(self._http_mocker, _BASE_URL, _STREAM_NAME, SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime")) + given_stream( + self._http_mocker, + _BASE_URL, + _STREAM_NAME, + SalesforceDescribeResponseBuilder().field(_A_FIELD_NAME).field(_CURSOR_FIELD, "datetime"), + ) def tearDown(self) -> None: self._http_mocker.__exit__(None, None, None) @@ -102,11 +110,13 @@ def tearDown(self) -> None: def test_given_no_state_when_read_then_start_sync_from_start(self) -> None: start = _calculate_start_time(_NOW - timedelta(days=5)) # as the start comes from the config, we can't use the same format as `_to_url` - start_format_url = urllib.parse.quote_plus(start.strftime('%Y-%m-%dT%H:%M:%SZ')) + start_format_url = urllib.parse.quote_plus(start.strftime("%Y-%m-%dT%H:%M:%SZ")) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{start_format_url}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME], record_count=1), ) @@ -119,13 +129,22 @@ def test_given_sequential_state_when_read_then_migrate_to_partitioned_state(self start = _calculate_start_time(_NOW - timedelta(days=10)) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(cursor_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) - output = read(_STREAM_NAME, SyncMode.incremental, self._config, StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")})) + output = read( + _STREAM_NAME, + SyncMode.incremental, + self._config, + StateBuilder().with_stream_state(_STREAM_NAME, {_CURSOR_FIELD: cursor_value.isoformat(timespec="milliseconds")}), + ) - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_update_state(self) -> None: missing_chunk = (_NOW - timedelta(days=5), _NOW - timedelta(days=3)) @@ -138,21 +157,27 @@ def test_given_partitioned_state_when_read_then_sync_missing_partitions_and_upda "slices": [ {"start": start.strftime("%Y-%m-%dT%H:%M:%S.000") + "Z", "end": _to_partitioned_datetime(missing_chunk[0])}, {"start": _to_partitioned_datetime(missing_chunk[1]), "end": _to_partitioned_datetime(most_recent_state_value)}, - ] - } + ], + }, ) self._config.stream_slice_step("P30D").start_date(start) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(missing_chunk[0])}+AND+SystemModstamp+%3C+{_to_url(missing_chunk[1])}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) self._http_mocker.get( - HttpRequest(f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}"), + HttpRequest( + f"{_BASE_URL}/queryAll?q=SELECT+{_A_FIELD_NAME},{_CURSOR_FIELD}+FROM+{_STREAM_NAME}+WHERE+SystemModstamp+%3E%3D+{_to_url(most_recent_state_value - _LOOKBACK_WINDOW)}+AND+SystemModstamp+%3C+{_to_url(_NOW)}" + ), create_http_response([_A_FIELD_NAME, _CURSOR_FIELD], record_count=1), ) output = read(_STREAM_NAME, SyncMode.incremental, self._config, state) # the start is granular to the second hence why we have `000` in terms of milliseconds - assert output.most_recent_state.stream_state == AirbyteStateBlob({"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]}) + assert output.most_recent_state.stream_state == AirbyteStateBlob( + {"state_type": "date-range", "slices": [{"start": _to_partitioned_datetime(start), "end": _to_partitioned_datetime(_NOW)}]} + ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py index 3fa57dfbd0a1d..3aca87de79593 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_source.py @@ -29,9 +29,7 @@ class StreamGenerationTest(TestCase): def setUp(self) -> None: self._config = ConfigBuilder().client_id(_CLIENT_ID).client_secret(_CLIENT_SECRET).refresh_token(_REFRESH_TOKEN).build() self._source = SourceSalesforce( - CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), - self._config, - StateBuilder().build() + CatalogBuilder().with_stream(_STREAM_NAME, SyncMode.full_refresh).build(), self._config, StateBuilder().build() ) self._http_mocker = HttpMocker() @@ -48,10 +46,7 @@ def test_given_transient_error_fetching_schema_when_streams_then_retry(self) -> ) self._http_mocker.get( HttpRequest(f"{_BASE_URL}/sobjects/{_STREAM_NAME}/describe"), - [ - HttpResponse("", status_code=406), - SalesforceDescribeResponseBuilder().field("a_field_name").build() - ] + [HttpResponse("", status_code=406), SalesforceDescribeResponseBuilder().field("a_field_name").build()], ) streams = self._source.streams(self._config) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py index b547a297c439e..bf1f2eee471bc 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/integration/utils.py @@ -35,7 +35,7 @@ def read( sync_mode: SyncMode, config_builder: Optional[ConfigBuilder] = None, state_builder: Optional[StateBuilder] = None, - expecting_exception: bool = False + expecting_exception: bool = False, ) -> EntrypointOutput: catalog = _catalog(stream_name, sync_mode) config = config_builder.build() if config_builder else ConfigBuilder().build() @@ -43,12 +43,19 @@ def read( return entrypoint_read(_source(catalog, config, state), config, catalog, state, expecting_exception) -def given_authentication(http_mocker: HttpMocker, client_id: str, client_secret: str, refresh_token: str, instance_url: str, access_token: str = "any_access_token") -> None: +def given_authentication( + http_mocker: HttpMocker, + client_id: str, + client_secret: str, + refresh_token: str, + instance_url: str, + access_token: str = "any_access_token", +) -> None: http_mocker.post( HttpRequest( "https://login.salesforce.com/services/oauth2/token", query_params=ANY_QUERY_PARAMS, - body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}" + body=f"grant_type=refresh_token&client_id={client_id}&client_secret={client_secret}&refresh_token={refresh_token}", ), HttpResponse(json.dumps({"access_token": access_token, "instance_url": instance_url})), ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py index 55bc8b8f65ddf..76af9b7eaf382 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py @@ -9,18 +9,18 @@ class JobCreateResponseBuilder: def __init__(self) -> None: self._response = { - "id": "any_id", - "operation": "query", - "object": "Account", - "createdById": "005R0000000GiwjIAC", - "createdDate": "2018-12-17T21:00:17.000+0000", - "systemModstamp": "2018-12-17T21:00:17.000+0000", - "state": "UploadComplete", - "concurrencyMode": "Parallel", - "contentType": "CSV", - "apiVersion": 46.0, - "lineEnding": "LF", - "columnDelimiter": "COMMA" + "id": "any_id", + "operation": "query", + "object": "Account", + "createdById": "005R0000000GiwjIAC", + "createdDate": "2018-12-17T21:00:17.000+0000", + "systemModstamp": "2018-12-17T21:00:17.000+0000", + "state": "UploadComplete", + "concurrencyMode": "Parallel", + "contentType": "CSV", + "apiVersion": 46.0, + "lineEnding": "LF", + "columnDelimiter": "COMMA", } self._status_code = 200 @@ -52,11 +52,11 @@ def with_state(self, state: str) -> "JobInfoResponseBuilder": def with_status_code(self, status_code: int) -> "JobInfoResponseBuilder": self._status_code = status_code return self - + def with_error_message(self, error_message: str) -> "JobInfoResponseBuilder": self._response["errorMessage"] = error_message return self - + def get_response(self) -> any: return self._response diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py index c6b9d1ea17317..c8a7a38fb76d1 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_rate_limiting.py @@ -20,18 +20,32 @@ class SalesforceErrorHandlerTest(TestCase): def setUp(self) -> None: self._error_handler = SalesforceErrorHandler() - def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}]) + def test_given_invalid_entity_with_bulk_not_supported_message_on_job_creation_when_interpret_response_then_raise_bulk_not_supported( + self, + ) -> None: + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "INVALIDENTITY", "message": "X is not supported by the Bulk API"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_compound_data_error_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": _ANY, "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_request_limit_exceeded_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}]) + response = self._create_response( + "POST", + self._url_for_job_creation(), + 400, + [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "Selecting compound data not supported in Bulk Query"}], + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) @@ -41,12 +55,24 @@ def test_given_limit_exceeded_on_job_creation_when_interpret_response_then_raise self._error_handler.interpret_response(response) def test_given_query_not_supported_on_job_creation_when_interpret_response_then_raise_bulk_not_supported(self) -> None: - response = self._create_response("POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}]) + response = self._create_response( + "POST", self._url_for_job_creation(), 400, [{"errorCode": "API_ERROR", "message": "API does not support query"}] + ) with pytest.raises(BulkNotSupportedException): self._error_handler.interpret_response(response) def test_given_txn_security_metering_error_when_interpret_response_then_raise_config_error(self) -> None: - response = self._create_response("GET", self._url_for_job_creation() + "/job_id", 400, [{"errorCode": "TXN_SECURITY_METERING_ERROR", "message": "We can't complete the action because enabled transaction security policies took too long to complete."}]) + response = self._create_response( + "GET", + self._url_for_job_creation() + "/job_id", + 400, + [ + { + "errorCode": "TXN_SECURITY_METERING_ERROR", + "message": "We can't complete the action because enabled transaction security policies took too long to complete.", + } + ], + ) error_resolution = self._error_handler.interpret_response(response) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py index 25ea70f6362cf..6f95018a17696 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_slice_generation.py @@ -12,12 +12,14 @@ _NOW = datetime.fromisoformat("2020-01-01T00:00:00+00:00") _STREAM_NAME = UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] + @freezegun.freeze_time(time_to_freeze=_NOW) class IncrementalSliceGenerationTest(TestCase): """ For this, we will be testing with UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS[0] as bulk stream slicing actually creates jobs. We will assume the bulk one usese the same logic. """ + def test_given_start_within_slice_range_when_stream_slices_then_return_one_slice_considering_10_minutes_lookback(self) -> None: config = ConfigBuilder().start_date(_NOW - timedelta(days=15)).stream_slice_step("P30D").build() stream = generate_stream(_STREAM_NAME, config, mock_stream_api(config)) @@ -34,5 +36,5 @@ def test_given_slice_range_smaller_than_now_minus_start_date_when_stream_slices_ assert slices == [ {"start_date": "2019-11-22T00:00:00.000+00:00", "end_date": "2019-12-22T00:00:00.000+00:00"}, - {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"} + {"start_date": "2019-12-22T00:00:00.000+00:00", "end_date": "2020-01-01T00:00:00.000+00:00"}, ] diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py index 0c5af692c5af2..ab243cf755d66 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/config_builder.py @@ -10,7 +10,7 @@ def __init__(self) -> None: "auth_token": "test token", "organization": "test organization", "project": "test project", - "hostname": "sentry.io" + "hostname": "sentry.io", } def build(self) -> Dict[str, Any]: diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py index 85987dd3d2dde..10699c214b0cd 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_events_stream.py @@ -29,12 +29,8 @@ def state(self): @HttpMocker() def test_read(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() @@ -47,12 +43,8 @@ def test_read(self, http_mocker: HttpMocker): @HttpMocker() def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( - HttpRequest( - url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", - query_params={"full": "true"} - ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpRequest(url="https://sentry.io/api/0/projects/test%20organization/test%20project/events/", query_params={"full": "true"}), + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py index 0103ab4856e8c..d6d54ee01d941 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/integration/test_issues_stream.py @@ -31,10 +31,9 @@ def test_read(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>1900-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.fr_read_file, __file__)), status_code=200), ) # https://sentry.io/api/1/projects/airbyte-09/airbyte-09/issues/?query=lastSeen%3A%3E2022-01-01T00%3A00%3A00.0Z config = self.config() @@ -50,10 +49,9 @@ def test_read_incremental(self, http_mocker: HttpMocker): http_mocker.get( HttpRequest( url="https://sentry.io/api/0/projects/test%20organization/test%20project/issues/", - query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"} + query_params={"query": "lastSeen:>2023-01-01T00:00:00.000000Z"}, ), - HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200) - + HttpResponse(body=json.dumps(find_template(self.inc_read_file, __file__)), status_code=200), ) config = self.config() catalog = self.catalog() diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 5f3c1f349ebd8..e5395848ec3ea 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -24,7 +24,10 @@ def test_next_page_token(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": "next-page"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == "next-page" + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == "next-page" + ) def test_next_page_token_is_none(): @@ -33,7 +36,9 @@ def test_next_page_token_is_none(): response_mock.headers = {} # stop condition: "results": "false" response_mock.links = {"next": {"cursor": "", "results": "false"}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) is None + ) def test_events_path(): @@ -77,7 +82,10 @@ def test_projects_request_params(): response_mock = MagicMock() response_mock.headers = {} response_mock.links = {"next": {"cursor": expected}} - assert stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) == expected + assert ( + stream.retriever.paginator.pagination_strategy.next_page_token(response=response_mock, last_page_size=0, last_record=None) + == expected + ) def test_project_detail_request_params(): @@ -89,10 +97,7 @@ def test_project_detail_request_params(): def test_project_detail_parse_response(requests_mock): expected = {"id": "1", "name": "test project"} stream = get_stream_by_name("project_detail") - requests_mock.get( - "https://sentry.io/api/0/projects/test-org/test-project/", - json=expected - ) + requests_mock.get("https://sentry.io/api/0/projects/test-org/test-project/", json=expected) result = list(stream.read_records(sync_mode=SyncMode.full_refresh))[0] assert expected == result.data @@ -137,4 +142,3 @@ def test_issues_validate_state_value(state, expected): stream = get_stream_by_name("issues") stream.retriever.state = state assert stream.state.get(stream.cursor_field) == expected - diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py index f6e0b7560c29c..317d57f41fdfe 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py @@ -99,18 +99,26 @@ def test_get_files_empty_files(configured_catalog: ConfiguredAirbyteCatalog, con def test_get_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_file_transfer: Mapping[str, Any]): source = SourceSFTPBulk(catalog=configured_catalog, config=config_fixture_use_file_transfer, state=None) output = read(source=source, config=config_fixture_use_file_transfer, catalog=configured_catalog) - expected_file_data = {'bytes': 46_754_266, 'file_relative_path': 'files/file_transfer/file_transfer_1.csv', 'file_url': '/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv', 'modified': ANY, 'source_file_url': '/files/file_transfer/file_transfer_1.csv'} + expected_file_data = { + "bytes": 46_754_266, + "file_relative_path": "files/file_transfer/file_transfer_1.csv", + "file_url": "/tmp/airbyte-file-transfer/files/file_transfer/file_transfer_1.csv", + "modified": ANY, + "source_file_url": "/files/file_transfer/file_transfer_1.csv", + } assert len(output.records) == 1 assert list(map(lambda record: record.record.file, output.records)) == [expected_file_data] # Additional assertion to check if the file exists at the file_url path - file_path = expected_file_data['file_url'] + file_path = expected_file_data["file_url"] assert os.path.exists(file_path), f"File not found at path: {file_path}" @pytest.mark.slow @pytest.mark.limit_memory("10 MB") -def test_get_all_file_csv_file_transfer(configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any]): +def test_get_all_file_csv_file_transfer( + configured_catalog: ConfiguredAirbyteCatalog, config_fixture_use_all_files_transfer: Mapping[str, Any] +): """ - The Paramiko dependency `get` method uses requests parallelization for efficiency, which may slightly increase memory usage. - The test asserts that this memory increase remains below the files sizes being transferred. diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py index 6d53197ff18a8..3d955fd8b5ec2 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -25,7 +25,6 @@ def __init__(self, auth_method: str = None): class ShopifyAuthenticator(TokenAuthenticator): - """ Making Authenticator to be able to accept Header-Based authentication. """ diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py index 2751a3ab9756c..193d1c9d47d1b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py @@ -249,7 +249,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream): class BalanceTransactions(IncrementalShopifyStream): - """ PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: https://shopify.dev/api/admin-rest/2021-07/resources/transactions diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index d7fd17a428464..a6620f45a1df0 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -47,7 +47,7 @@ def __new__(self, stream: str) -> Mapping[str, Any]: response_action=ResponseAction.IGNORE, failure_type=FailureType.config_error, error_message=f"Stream `{stream}`. Entity might not be available or missing.", - ) + ), # extend the mapping with more handable errors, if needed. } diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index 0f296c87061a3..abf947558d104 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -39,7 +39,7 @@ def logger(): @pytest.fixture def basic_config(): return { - "shop": "test_shop", + "shop": "test_shop", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "shop_id": 0, } @@ -52,7 +52,6 @@ def auth_config(): "start_date": "2023-01-01", "credentials": {"auth_method": "api_password", "api_password": "api_password"}, "authenticator": None, - } @@ -358,7 +357,7 @@ def bulk_job_failed_response(): }, } - + @pytest.fixture def bulk_job_failed_with_partial_url_response(): return { @@ -371,20 +370,16 @@ def bulk_job_failed_with_partial_url_response(): "fileSize": None, "url": None, "partialDataUrl": 'https://some_url?response-content-disposition=attachment;+filename="bulk-123456789.jsonl";+filename*=UTF-8' - "bulk-123456789.jsonl&response-content-type=application/jsonl" + "bulk-123456789.jsonl&response-content-type=application/jsonl", } }, "extensions": { "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 20000.0, - "currentlyAvailable": 19999, - "restoreRate": 1000.0 - } + "throttleStatus": {"maximumAvailable": 20000.0, "currentlyAvailable": 19999, "restoreRate": 1000.0}, } - } + }, } @@ -442,8 +437,8 @@ def bulk_job_running_response(): } }, } - - + + @pytest.fixture def bulk_job_running_with_object_count_and_url_response(): return { diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py index 90eb6faaf67db..cf34bf5a5107e 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py @@ -31,9 +31,9 @@ def test_job_manager_default_values(auth_config) -> None: stream = Products(auth_config) - + # 10Mb chunk size to save the file - assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 + assert stream.job_manager._retrieve_chunk_size == 10485760 # 1024 * 1024 * 10 assert stream.job_manager._job_max_retries == 6 assert stream.job_manager._job_backoff_time == 5 # running job logger constrain, every 100-ish message will be printed @@ -48,7 +48,7 @@ def test_job_manager_default_values(auth_config) -> None: # currents: _job_id, _job_state, _job_created_at, _job_self_canceled assert not stream.job_manager._job_id # this string is based on ShopifyBulkJobStatus - assert not stream.job_manager._job_state + assert not stream.job_manager._job_state # completed and saved Bulk Job result filename assert not stream.job_manager._job_result_filename # date-time when the Bulk Job was created on the server @@ -56,7 +56,7 @@ def test_job_manager_default_values(auth_config) -> None: # indicated whether or not we manually force-cancel the current job assert not stream.job_manager._job_self_canceled # time between job status checks - assert stream.job_manager. _job_check_interval == 3 + assert stream.job_manager._job_check_interval == 3 # 0.1 ~= P2H, default value, lower boundary for slice size assert stream.job_manager._job_size_min == 0.1 # last running job object count @@ -99,8 +99,9 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: {"json": request.getfixturevalue("bulk_error_with_concurrent_job")}, # concurrent request has finished {"json": request.getfixturevalue("bulk_successful_response")}, - ]) - + ], + ) + stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) # call count should be 4 (3 retries, 1 - succeeded) assert requests_mock.call_count == 4 @@ -119,14 +120,16 @@ def test_retry_on_concurrent_job(request, requests_mock, auth_config) -> None: ], ids=[ "max attempt reached", - ] + ], ) -def test_job_retry_on_concurrency(request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected) -> None: +def test_job_retry_on_concurrency( + request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # patching concurrent settings stream.job_manager._concurrent_max_retry = concurrent_max_retry stream.job_manager._concurrent_interval = 1 - + requests_mock.post(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) if error_type: @@ -200,8 +203,8 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, mocker.patch("source_shopify.shopify_graphql.bulk.record.ShopifyBulkRecord.read_file", return_value=[]) stream.job_manager._job_check_state() assert expected == stream.job_manager._job_result_filename - - + + @pytest.mark.parametrize( "job_response, error_type, expected", [ @@ -211,7 +214,9 @@ def test_job_check_for_completion(mocker, request, requests_mock, job_response, "failed", ], ) -def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, requests_mock, job_response, error_type, expected, auth_config) -> None: +def test_job_failed_for_stream_with_no_bulk_checkpointing( + mocker, request, requests_mock, job_response, error_type, expected, auth_config +) -> None: stream = InventoryLevels(auth_config) # modify the sleep time for the test stream.job_manager._concurrent_max_retry = 1 @@ -253,26 +258,28 @@ def test_job_failed_for_stream_with_no_bulk_checkpointing(mocker, request, reque "BulkJobBadResponse", ], ) -def test_retry_on_job_creation_exception(request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg) -> None: +def test_retry_on_job_creation_exception( + request, requests_mock, auth_config, job_response, job_state, error_type, max_retry, call_count_expected, expected_msg +) -> None: stream = MetafieldOrders(auth_config) stream.job_manager._job_backoff_time = 0 stream.job_manager._job_max_retries = max_retry # patching the method to get the right ID checks if job_response: stream.job_manager._job_id = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("id") - + if job_state: # setting job_state to simulate the error-in-the-middle stream.job_manager._job_state = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("status") - + # mocking the response for STATUS CHECKS json_mock_response = request.getfixturevalue(job_response) if job_response else None requests_mock.post(stream.job_manager.base_url, json=json_mock_response) - + # testing raised exception and backoff with pytest.raises(error_type) as error: stream.job_manager.create_job(_ANY_SLICE, _ANY_FILTER_FIELD) - + # we expect different call_count, because we set the different max_retries assert expected_msg in repr(error.value) and requests_mock.call_count == call_count_expected @@ -302,11 +309,11 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a job_result_url = test_job_status_response.json().get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + # mocking the nested request call to retrieve the data from result URL stream.job_manager._job_id = job_id requests_mock.get(job_result_url, json=request.getfixturevalue(job_response)) - + # calling the sceario processing stream.job_manager._job_track_running() assert stream.job_manager._job_state == expected @@ -316,13 +323,13 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "running_job_response, canceled_job_response, expected", [ ( - "bulk_job_running_with_object_count_and_url_response", - "bulk_job_canceled_with_object_count_and_url_response", + "bulk_job_running_with_object_count_and_url_response", + "bulk_job_canceled_with_object_count_and_url_response", "bulk-123456789.jsonl", ), ( - "bulk_job_running_with_object_count_no_url_response", - "bulk_job_canceled_with_object_count_no_url_response", + "bulk_job_running_with_object_count_no_url_response", + "bulk_job_canceled_with_object_count_no_url_response", None, ), ], @@ -331,7 +338,9 @@ def test_job_check_with_running_scenario(request, requests_mock, job_response, a "self-canceled with no url", ], ) -def test_job_running_with_canceled_scenario(mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected) -> None: +def test_job_running_with_canceled_scenario( + mocker, request, requests_mock, running_job_response, canceled_job_response, auth_config, expected +) -> None: stream = MetafieldOrders(auth_config) # modify the sleep time for the test stream.job_manager._job_check_interval = 0 @@ -339,7 +348,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_id = request.getfixturevalue(running_job_response).get("data", {}).get("node", {}).get("id") # mocking the response for STATUS CHECKS requests_mock.post( - stream.job_manager.base_url, + stream.job_manager.base_url, [ {"json": request.getfixturevalue(running_job_response)}, {"json": request.getfixturevalue(canceled_job_response)}, @@ -348,7 +357,7 @@ def test_job_running_with_canceled_scenario(mocker, request, requests_mock, runn job_result_url = request.getfixturevalue(canceled_job_response).get("data", {}).get("node", {}).get("url") # test the state of the job isn't assigned assert stream.job_manager._job_state == None - + stream.job_manager._job_id = job_id stream.job_manager._job_checkpoint_interval = 5 # faking self-canceled job @@ -448,9 +457,9 @@ def test_bulk_stream_parse_response( ) def test_stream_slices( auth_config, - stream, - stream_state, - with_start_date, + stream, + stream_state, + with_start_date, expected_start, ) -> None: # simulating `None` for `start_date` and `config migration` @@ -462,7 +471,7 @@ def test_stream_slices( test_result = list(stream.stream_slices(stream_state=stream_state)) assert test_result[0].get("start") == expected_start - + @pytest.mark.parametrize( "stream, json_content_example, last_job_elapsed_time, previous_slice_size, adjusted_slice_size", [ @@ -471,7 +480,7 @@ def test_stream_slices( ids=[ "Expand Slice Size", ], -) +) def test_expand_stream_slices_job_size( request, requests_mock, @@ -483,7 +492,7 @@ def test_expand_stream_slices_job_size( adjusted_slice_size, auth_config, ) -> None: - + stream = stream(auth_config) # get the mocked job_result_url test_result_url = bulk_job_completed_response.get("data").get("node").get("url") @@ -495,7 +504,7 @@ def test_expand_stream_slices_job_size( # for the sake of simplicity we fake some parts to simulate the `current_job_time_elapsed` # fake current slice interval value stream.job_manager._job_size = previous_slice_size - # fake `last job elapsed time` + # fake `last job elapsed time` if last_job_elapsed_time: stream.job_manager._job_last_elapsed_time = last_job_elapsed_time diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py index 8a5d7012d38bc..cf586b159ae17 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -30,12 +30,12 @@ def test_query_status() -> None: } } }""" - + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.status(input_job_id) assert repr(template) == repr(expected) - - + + def test_bulk_query_prepare() -> None: expected = '''mutation { bulkOperationRunQuery( @@ -54,14 +54,14 @@ def test_bulk_query_prepare() -> None: } } }''' - + input_query_from_slice = "{some_query}" template = ShopifyBulkTemplates.prepare(input_query_from_slice) assert repr(template) == repr(expected) - - + + def test_bulk_query_cancel() -> None: - expected = '''mutation { + expected = """mutation { bulkOperationCancel(id: "gid://shopify/BulkOperation/4047052112061") { bulkOperation { id @@ -73,32 +73,32 @@ def test_bulk_query_cancel() -> None: message } } - }''' - + }""" + input_job_id = "gid://shopify/BulkOperation/4047052112061" template = ShopifyBulkTemplates.cancel(input_job_id) assert repr(template) == repr(expected) - + @pytest.mark.parametrize( "query_name, fields, filter_field, start, end, expected", [ ( - "test_root", - ["test_field1", "test_field2"], + "test_root", + ["test_field1", "test_field2"], "updated_at", "2023-01-01", - "2023-01-02", + "2023-01-02", Query( - name='test_root', + name="test_root", arguments=[ - Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=["test_field1", "test_field2"])])] - ) + Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), + ], + fields=[Field(name="edges", fields=[Field(name="node", fields=["test_field1", "test_field2"])])], + ), ) ], - ids=["simple query with filter and sort"] + ids=["simple query with filter and sort"], ) def test_base_build_query(basic_config, query_name, fields, filter_field, start, end, expected) -> None: """ @@ -116,7 +116,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, } ''' """ - + builder = ShopifyBulkQuery(basic_config) filter_query = f"{filter_field}:>'{start}' AND {filter_field}:<='{end}'" built_query = builder.build(query_name, fields, filter_query) @@ -136,14 +136,52 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='customers', + name="customers", arguments=[ Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - Argument(name="sortKey", value="UPDATED_AT"), - ], - fields=[Field(name='edges', fields=[Field(name='node', fields=['__typename', 'id', Field(name="updatedAt", alias="customers_updated_at"), Field(name="metafields", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", "namespace", "value", "key", "description", "createdAt", "updatedAt", "type"])])])])])] + Argument(name="sortKey", value="UPDATED_AT"), + ], + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + Field(name="updatedAt", alias="customers_updated_at"), + Field( + name="metafields", + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + "namespace", + "value", + "key", + "description", + "createdAt", + "updatedAt", + "type", + ], + ) + ], + ) + ], + ), + ], + ) + ], + ) + ], ) - ] + ], ), ), ( @@ -206,28 +244,28 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "createdAt", "updatedAt", "type", - ] + ], ) - ] + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] - ) + ], + ), ), ( InventoryLevel, @@ -239,64 +277,91 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, type="", queries=[ Query( - name='locations', + name="locations", arguments=[ Argument(name="includeLegacy", value="true"), Argument(name="includeInactive", value="true"), - ], + ], fields=[ Field( - name='edges', + name="edges", fields=[ Field( - name='node', + name="node", fields=[ - '__typename', - 'id', + "__typename", + "id", Field( - name="inventoryLevels", + name="inventoryLevels", arguments=[ - Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), - ], + Argument( + name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\"" + ), + ], fields=[ Field( - name="edges", + name="edges", fields=[ Field( - name="node", + name="node", fields=[ - "__typename", - "id", - "canDeactivate", - "createdAt", - "deactivationAlert", - "updatedAt", - Field(name="item", fields=[Field(name="inventoryHistoryUrl", alias="inventory_history_url"), Field(name="id", alias="inventory_item_id"), Field(name="locationsCount", alias="locations_count", fields=["count"])]), - Field( - name="quantities", + "__typename", + "id", + "canDeactivate", + "createdAt", + "deactivationAlert", + "updatedAt", + Field( + name="item", + fields=[ + Field( + name="inventoryHistoryUrl", alias="inventory_history_url" + ), + Field(name="id", alias="inventory_item_id"), + Field( + name="locationsCount", + alias="locations_count", + fields=["count"], + ), + ], + ), + Field( + name="quantities", arguments=[ - Argument(name="names", value=['"available"', '"incoming"', '"committed"', '"damaged"', '"on_hand"', '"quality_control"', '"reserved"', '"safety_stock"']) - ], + Argument( + name="names", + value=[ + '"available"', + '"incoming"', + '"committed"', + '"damaged"', + '"on_hand"', + '"quality_control"', + '"reserved"', + '"safety_stock"', + ], + ) + ], fields=[ "id", "name", "quantity", "updatedAt", ], - ) - ] + ), + ], ) - ] + ], ) - ] - ) - ] + ], + ), + ], ) - ] + ], ) - ] + ], ) - ] + ], ), ), ], @@ -304,7 +369,7 @@ def test_base_build_query(basic_config, query_name, fields, filter_field, start, "MetafieldCustomers query with 1 query_path(str)", "MetafieldProductImages query with composite quey_path(List[2])", "InventoryLevel query", - ] + ], ) def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_stream_class, expected) -> None: if parent_stream_class: @@ -314,4 +379,4 @@ def test_bulk_query(auth_config, query_class, filter_field, start, end, parent_s else: stream_query = query_class(auth_config) - assert stream_query.get(filter_field, start, end) == expected.render() \ No newline at end of file + assert stream_query.get(filter_field, start, end) == expected.render() diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py index dff60ea605d59..85c48ce6d13b3 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py @@ -58,7 +58,7 @@ def test_check_type(basic_config, record, types, expected) -> None: "alias_to_id_field": "gid://shopify/Metafield/123", "__parentId": "gid://shopify/Order/102030", }, - ) + ), ], ) def test_record_resolver(basic_config, record, expected) -> None: diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py index 05b15d05dc008..5fef88d85a986 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/api/bulk.py @@ -42,16 +42,15 @@ def create_job_creation_body(lower_boundary: datetime, upper_boundary: datetime) } } }""" - query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace("%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat()) + query = query.replace("%LOWER_BOUNDARY_TOKEN%", lower_boundary.isoformat()).replace( + "%UPPER_BOUNDARY_TOKEN%", upper_boundary.isoformat() + ) prepared_query = ShopifyBulkTemplates.prepare(query) return json.dumps({"query": prepared_query}) def create_job_creation_request(shop_name: str, lower_boundary: datetime, upper_boundary: datetime) -> HttpRequest: - return HttpRequest( - url=_create_job_url(shop_name), - body=create_job_creation_body(lower_boundary, upper_boundary) - ) + return HttpRequest(url=_create_job_url(shop_name), body=create_job_creation_body(lower_boundary, upper_boundary)) def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: @@ -70,7 +69,7 @@ def create_job_status_request(shop_name: str, job_id: str) -> HttpRequest: partialDataUrl }} }} - }}""" + }}""", ) @@ -89,33 +88,26 @@ def create_job_cancel_request(shop_name: str, job_id: str) -> HttpRequest: message }} }} - }}""" + }}""", ) + class JobCreationResponseBuilder: - def __init__(self, job_created_at:str="2024-05-05T02:00:00Z") -> None: + def __init__(self, job_created_at: str = "2024-05-05T02:00:00Z") -> None: self._template = { "data": { "bulkOperationRunQuery": { - "bulkOperation": { - "id": "gid://shopify/BulkOperation/0", - "status": "CREATED", - "createdAt": f"{job_created_at}" - }, - "userErrors": [] + "bulkOperation": {"id": "gid://shopify/BulkOperation/0", "status": "CREATED", "createdAt": f"{job_created_at}"}, + "userErrors": [], } }, "extensions": { "cost": { "requestedQueryCost": 10, "actualQueryCost": 10, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1990, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1990, "restoreRate": 100.0}, } - } + }, } def with_bulk_operation_id(self, bulk_operation_id: str) -> "JobCreationResponseBuilder": @@ -135,30 +127,26 @@ def __init__(self) -> None: "cost": { "requestedQueryCost": 1, "actualQueryCost": 1, - "throttleStatus": { - "maximumAvailable": 2000.0, - "currentlyAvailable": 1999, - "restoreRate": 100.0 - } + "throttleStatus": {"maximumAvailable": 2000.0, "currentlyAvailable": 1999, "restoreRate": 100.0}, } - } + }, } } - def with_running_status(self, bulk_operation_id: str, object_count: str="10") -> "JobStatusResponseBuilder": + def with_running_status(self, bulk_operation_id: str, object_count: str = "10") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { - "id": bulk_operation_id, - "status": "RUNNING", - "errorCode": None, - "createdAt": "2024-05-28T18:57:54Z", - "objectCount": object_count, - "fileSize": None, - "url": None, - "partialDataUrl": None, + "id": bulk_operation_id, + "status": "RUNNING", + "errorCode": None, + "createdAt": "2024-05-28T18:57:54Z", + "objectCount": object_count, + "fileSize": None, + "url": None, + "partialDataUrl": None, } return self - def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_completed_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "COMPLETED", @@ -167,11 +155,11 @@ def with_completed_status(self, bulk_operation_id: str, job_result_url: str, obj "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self - def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str="4") -> "JobStatusResponseBuilder": + def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, object_count: str = "4") -> "JobStatusResponseBuilder": self._template["data"]["node"] = { "id": bulk_operation_id, "status": "CANCELED", @@ -180,7 +168,7 @@ def with_canceled_status(self, bulk_operation_id: str, job_result_url: str, obje "objectCount": object_count, "fileSize": "774", "url": job_result_url, - "partialDataUrl": None + "partialDataUrl": None, } return self @@ -192,15 +180,14 @@ class MetafieldOrdersJobResponseBuilder: def __init__(self) -> None: self._records = [] - def _any_record(self, updated_at:str="2024-05-05T01:09:50Z") -> str: + def _any_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> str: an_id = str(randint(1000000000000, 9999999999999)) a_parent_id = str(randint(1000000000000, 9999999999999)) return f"""{{"__typename":"Order","id":"gid:\/\/shopify\/Order\/{a_parent_id}"}} {{"__typename":"Metafield","id":"gid:\/\/shopify\/Metafield\/{an_id}","namespace":"my_fields","value":"asdfasdf","key":"purchase_order","description":null,"createdAt":"2023-04-13T12:09:50Z","updatedAt":"{updated_at}","type":"single_line_text_field","__parentId":"gid:\/\/shopify\/Order\/{a_parent_id}"}} """ - - def with_record(self, updated_at:str="2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": + def with_record(self, updated_at: str = "2024-05-05T01:09:50Z") -> "MetafieldOrdersJobResponseBuilder": self._records.append(self._any_record(updated_at=updated_at)) return self diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py index 8a0d6d5c8c59b..18f59a4a7f165 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/integration/test_bulk_stream.py @@ -41,6 +41,7 @@ _INCREMENTAL_JOB_START_DATE = datetime.fromisoformat(_INCREMENTAL_JOB_START_DATE_ISO) _INCREMENTAL_JOB_END_DATE = _INCREMENTAL_JOB_START_DATE + timedelta(hours=24, minutes=0) + def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_interval=200000) -> Dict[str, Any]: return { "start_date": start_date.strftime("%Y-%m-%d"), @@ -50,7 +51,7 @@ def _get_config(start_date: datetime, bulk_window: int = 1, job_checkpoint_inter "api_password": "api_password", }, "bulk_window_in_days": bulk_window, - "job_checkpoint_interval": job_checkpoint_interval + "job_checkpoint_interval": job_checkpoint_interval, } @@ -103,8 +104,10 @@ def test_given_response_is_not_json_on_job_creation_when_read_then_retry(self) - job_creation_request, [ HttpResponse("This is not json"), - JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build(), # This will never get called (see assertion below) - ] + JobCreationResponseBuilder() + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), # This will never get called (see assertion below) + ], ) self._http_mocker.post( @@ -126,8 +129,11 @@ def test_given_connection_error_on_job_creation_when_read_then_retry_job_creatio inner_mocker.register_uri( # TODO the testing library should have the ability to generate ConnectionError. As this might not be trivial, we will wait for another case before implementing "POST", _URL_GRAPHQL, - [{"exc": ConnectionError("ConnectionError")}, {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}], - additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE) + [ + {"exc": ConnectionError("ConnectionError")}, + {"text": JobCreationResponseBuilder().with_bulk_operation_id(_BULK_OPERATION_ID).build().body, "status_code": 200}, + ], + additional_matcher=lambda request: request.text == create_job_creation_body(_JOB_START_DATE, _JOB_END_DATE), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -152,7 +158,7 @@ def test_given_retryable_error_on_first_get_job_status_when_read_then_retry(self [ _AN_ERROR_RESPONSE, JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -175,7 +181,7 @@ def test_given_retryable_error_on_get_job_status_when_read_then_retry(self) -> N JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID).build(), HttpResponse(json.dumps({"errors": ["an error"]})), JobStatusResponseBuilder().with_completed_status(_BULK_OPERATION_ID, _JOB_RESULT_URL).build(), - ] + ], ) self._http_mocker.get( HttpRequest(_JOB_RESULT_URL), @@ -209,7 +215,9 @@ def test_when_read_then_extract_records(self) -> None: job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=5) self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) self._http_mocker.post( create_job_status_request(_SHOP_NAME, _BULK_OPERATION_ID), @@ -220,14 +228,10 @@ def test_when_read_then_extract_records(self) -> None: MetafieldOrdersJobResponseBuilder().with_record().with_record().build(), ) # expectation is job start date should be the updated_at in orders - metafield_orders_orders_state = {"orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + metafield_orders_orders_state = { + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago @@ -238,13 +242,13 @@ def test_when_read_then_extract_records(self) -> None: assert len(output.records) == 2 def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(self) -> None: - """" + """ " The motivation of this test is https://github.com/airbytehq/oncall/issues/6874 - + In this scenario we end having stream_slices method to generate same slice N times. Our checkpointing logic will trigger when job_checkpoint_interval is passed, but there may be the case that such checkpoint has the same value as the current slice start date so we would end requesting same job. - + In this test: 1. First job requires to checkpoint as we pass the 1500 limit, it cancels the bulk job and checkpoints from last cursor value. 2. Next job just goes "fine". @@ -261,6 +265,7 @@ def test_when_read_with_updated_at_field_before_bulk_request_window_start_date(s {"type":"LOG","log":{"level":"INFO","message":"Stream: `metafield_orders`, the BULK Job: `gid://shopify/BulkOperation/4472588009771` is CREATED"}} ... """ + def add_n_records(builder, n, record_date: Optional[str] = None): for _ in range(n): builder = builder.with_record(updated_at=record_date) @@ -271,7 +276,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # create a job request self._http_mocker.post( create_job_creation_request(_SHOP_NAME, _INCREMENTAL_JOB_START_DATE, _INCREMENTAL_JOB_END_DATE), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(_BULK_OPERATION_ID).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(_BULK_OPERATION_ID) + .build(), ) # get job status self._http_mocker.post( @@ -282,15 +289,10 @@ def add_n_records(builder, n, record_date: Optional[str] = None): JobStatusResponseBuilder().with_running_status(_BULK_OPERATION_ID, object_count="16000").build(), # this will complete the job JobStatusResponseBuilder().with_canceled_status(_BULK_OPERATION_ID, _JOB_RESULT_URL, object_count="1700").build(), - ] + ], ) # mock the cancel operation request as we passed the 15000 rows - self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), - [ - HttpResponse(json.dumps({}), status_code=200) - ] - ) + self._http_mocker.post(create_job_cancel_request(_SHOP_NAME, _BULK_OPERATION_ID), [HttpResponse(json.dumps({}), status_code=200)]) # get results for the request that got cancelled adjusted_checkpoint_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(days=2, hours=6, minutes=30) adjusted_record_date = adjusted_checkpoint_start_date.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -309,7 +311,9 @@ def add_n_records(builder, n, record_date: Optional[str] = None): self._http_mocker.post( # The start date is caused by record date in previous iteration create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date), - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693950.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693950.jsonl&response-content-type=application%2Fjsonl" @@ -317,8 +321,10 @@ def add_n_records(builder, n, record_date: Optional[str] = None): create_job_status_request(_SHOP_NAME, next_bulk_operation_id), [ # this will output the job is running - JobStatusResponseBuilder().with_completed_status(next_bulk_operation_id, next_job_result_url).build(), - ] + JobStatusResponseBuilder() + .with_completed_status(next_bulk_operation_id, next_job_result_url) + .build(), + ], ) # get results for the request that got cancelled self._http_mocker.get( @@ -333,12 +339,14 @@ def add_n_records(builder, n, record_date: Optional[str] = None): adjusted_checkpoint_end_date = adjusted_checkpoint_start_date + timedelta(days=1) job_created_at = _INCREMENTAL_JOB_END_DATE - timedelta(minutes=4) create_job_request = create_job_creation_request(_SHOP_NAME, adjusted_checkpoint_start_date, adjusted_checkpoint_end_date) - + self._http_mocker.post( create_job_request, - JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")).with_bulk_operation_id(next_bulk_operation_id).build(), + JobCreationResponseBuilder(job_created_at=job_created_at.strftime("%Y-%m-%dT%H:%M:%SZ")) + .with_bulk_operation_id(next_bulk_operation_id) + .build(), ) - + base_status_responses = [ JobStatusResponseBuilder().with_running_status(next_bulk_operation_id, object_count="500").build(), # this should make the job get canceled as it gets over 15000 rows @@ -346,23 +354,17 @@ def add_n_records(builder, n, record_date: Optional[str] = None): # this will complete the job JobStatusResponseBuilder().with_canceled_status(next_bulk_operation_id, next_job_result_url, object_count="1700").build(), ] - + n_times_to_loop = 4 responses_in_loop = base_status_responses * n_times_to_loop # get job status next_job_result_url = "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/bulk-operation-outputs/l6lersgk4i81iqc3n6iisywwtipb-final?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1715633149&Signature=oMjQelfAzUW%2FdulC3HbuBapbUriUJ%2Bc9%2FKpIIf954VTxBqKChJAdoTmWT9ymh%2FnCiHdM%2BeM%2FADz5siAC%2BXtHBWkJfvs%2F0cYpse0ueiQsw6R8gW5JpeSbizyGWcBBWkv5j8GncAnZOUVYDxRIgfxcPb8BlFxBfC3wsx%2F00v9D6EHbPpkIMTbCOAhheJdw9GmVa%2BOMqHGHlmiADM34RDeBPrvSo65f%2FakpV2LBQTEV%2BhDt0ndaREQ0MrpNwhKnc3vZPzA%2BliOGM0wyiYr9qVwByynHq8c%2FaJPPgI5eGEfQcyepgWZTRW5S0DbmBIFxZJLN6Nq6bJ2bIZWrVriUhNGx2g%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4476008693960.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4476008693960.jsonl&response-content-type=application%2Fjsonl" - - self._http_mocker.post( - create_job_status_request(_SHOP_NAME, next_bulk_operation_id), - responses_in_loop - ) + + self._http_mocker.post(create_job_status_request(_SHOP_NAME, next_bulk_operation_id), responses_in_loop) # mock the cancel operation request as we passed the 15000 rows self._http_mocker.post( - create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), - [ - HttpResponse(json.dumps({}), status_code=200) - ] + create_job_cancel_request(_SHOP_NAME, next_bulk_operation_id), [HttpResponse(json.dumps({}), status_code=200)] ) # get results @@ -371,35 +373,28 @@ def add_n_records(builder, n, record_date: Optional[str] = None): HttpRequest(next_job_result_url), add_n_records(MetafieldOrdersJobResponseBuilder(), 80, adjusted_record_date).build(), ) - + # ********* end of request mocking ************* metafield_orders_orders_state = { - "orders": { - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, - "deleted": { - "deleted_at": "" - } - }, - "updated_at": _INCREMENTAL_JOB_START_DATE_ISO - } + "orders": {"updated_at": _INCREMENTAL_JOB_START_DATE_ISO, "deleted": {"deleted_at": ""}}, + "updated_at": _INCREMENTAL_JOB_START_DATE_ISO, + } stream_state = StateBuilder().with_stream_state(_BULK_STREAM, metafield_orders_orders_state).build() # we are passing to config a start date let's set something "old" as happen in many sources like 2 years ago config_start_date = _INCREMENTAL_JOB_START_DATE - timedelta(weeks=104) - output = self._read(_get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state) + output = self._read( + _get_config(config_start_date, job_checkpoint_interval=15000), sync_mode=SyncMode.incremental, state=stream_state + ) expected_error_message = "The stream: `metafield_orders` checkpoint collision is detected." result = output.errors[0].trace.error.internal_message - # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` + # The result of the test should be the `ShopifyBulkExceptions.BulkJobCheckpointCollisionError` assert result is not None and expected_error_message in result - def _read(self, config, sync_mode=SyncMode.full_refresh, state: Optional[List[AirbyteStateMessage]] = None): catalog = CatalogBuilder().with_stream(_BULK_STREAM, sync_mode).build() output = read(SourceShopify(), config, catalog, state=state) return output - - - diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py index 3d599ce770b11..ea902f0be5676 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py @@ -108,7 +108,9 @@ def test_read_deleted_records(stream, requests_mock, deleted_records_json, expec stream = stream(config) deleted_records_url = stream.url_base + stream.deleted_events.path() requests_mock.get(deleted_records_url, json=deleted_records_json) - mocker.patch("source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json) + mocker.patch( + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json + ) assert list(stream.read_records(sync_mode=None)) == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py index a6c99f9c3c44b..285b816734ed4 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py @@ -7,10 +7,25 @@ @pytest.mark.parametrize( "page_size, filter_value, next_page_token, expected_query", [ - (100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), - (250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + ( + 100, + None, + None, + "query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 200, + "2027-07-11T13:07:45-07:00", + None, + "query {\n products(first: 200, query: \"updated_at:>'2027-07-11T13:07:45-07:00'\", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}", + ), + ( + 250, + "2027-07-11T13:07:45-07:00", + "end_cursor_value", + 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}', + ), ], ) def test_get_query_products(page_size, filter_value, next_page_token, expected_query): - assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query + assert get_query_products(page_size, "updatedAt", filter_value, next_page_token) == expected_query diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index f9ef2ba2f7cad..8b694bfbf502b 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -126,13 +126,13 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi else: result = stream.path() assert result == expected_path - - + + @pytest.mark.parametrize( "stream, parent_records, state_checkpoint_interval", [ ( - OrderRefunds, + OrderRefunds, [ {"id": 1, "refunds": [{"created_at": "2021-01-01T00:00:00+00:00"}]}, {"id": 2, "refunds": [{"created_at": "2021-02-01T00:00:00+00:00"}]}, @@ -145,10 +145,10 @@ def test_path_with_stream_slice_param(stream, stream_slice, expected_path, confi ], ) def test_stream_slice_nested_substream_buffering( - mocker, - config, - stream, - parent_records, + mocker, + config, + stream, + parent_records, state_checkpoint_interval, ) -> None: # making the stream instance @@ -156,7 +156,7 @@ def test_stream_slice_nested_substream_buffering( stream.state_checkpoint_interval = state_checkpoint_interval # simulating `read_records` for the `parent_stream` mocker.patch( - "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", + "source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=parent_records, ) # count how many slices we expect, based on the number of parent_records @@ -173,7 +173,7 @@ def test_stream_slice_nested_substream_buffering( # count total slices total_slices += 1 # check we have emitted complete number of slices - assert total_slices == total_slices_expected + assert total_slices == total_slices_expected def test_check_connection(config, mocker) -> None: @@ -212,11 +212,23 @@ def test_request_params(config, stream, expected) -> None: "last_record, current_state, expected", [ # no init state - ({"created_at": "2022-10-10T06:21:53-07:00"}, {}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is empty str - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": ""}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": ""}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # state is None - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": None}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), + ( + {"created_at": "2022-10-10T06:21:53-07:00"}, + {"created_at": None}, + {"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}, + ), # last rec cursor is None ({"created_at": None}, {"created_at": None}, {"created_at": "", "orders": {"updated_at": "", "deleted": {"deleted_at": ""}}}), # last rec cursor is empty str @@ -257,21 +269,19 @@ def test_get_shop_name(config, shop, expected) -> None: actual = source.get_shop_name(config) assert actual == expected + @pytest.mark.parametrize( "config, expected_stream_class", [ ({"fetch_transactions_user_id": False}, TransactionsGraphql), ({"fetch_transactions_user_id": True}, Transactions), ({}, TransactionsGraphql), - ], + ], ids=["don't fetch user_id", "fetch user id", "unset config value shouldn't fetch user_id"], ) def test_select_transactions_stream(config, expected_stream_class): config["shop"] = "test-store" - config["credentials"] = { - "auth_method": "api_password", - "api_password": "shppa_123" - } + config["credentials"] = {"auth_method": "api_password", "api_password": "shppa_123"} config["authenticator"] = ShopifyAuthenticator(config) source = SourceShopify() @@ -284,7 +294,7 @@ def test_select_transactions_stream(config, expected_stream_class): [ pytest.param([{"id": "12345"}], "12345", None, id="test_shop_name_exists"), pytest.param([], None, AirbyteTracedException, id="test_shop_name_does_not_exist"), - ], + ], ) def test_get_shop_id(config, read_records, expected_shop_id, expected_error): check_test = ConnectionCheckTest(config) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index ca9e306e66983..649e409e70561 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -100,8 +100,9 @@ def test_privileges_validation(requests_mock, fetch_transactions_user_id, basic_ "Internal Server Error for slice (500)", ], ) -def test_unavailable_stream(requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, - json_response: Mapping[str, Any]): +def test_unavailable_stream( + requests_mock, auth_config, stream, slice: Optional[Mapping[str, Any]], status_code: int, json_response: Mapping[str, Any] +): stream = stream(auth_config) url = stream.url_base + stream.path(stream_slice=slice) requests_mock.get(url=url, json=json_response, status_code=status_code) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 002a9ec96779d..d71c92494bbe0 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -16,13 +16,10 @@ def conversations_list(requests_mock): return requests_mock.register_uri( "GET", "https://slack.com/api/conversations.list?limit=1000&types=public_channel", - json={ - "channels": [ - {"id": "airbyte-for-beginners", "is_member": True}, - {"id": "good-reads", "is_member": True}] - }, + json={"channels": [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}]}, ) + def base_config() -> MutableMapping: return copy.deepcopy( { @@ -98,11 +95,27 @@ def invalid_config() -> MutableMapping: @pytest.fixture def joined_channel(): - return {"id": "C061EG9SL", "name": "general", "is_channel": True, "is_group": False, "is_im": False, - "created": 1449252889, - "creator": "U061F7AUR", "is_archived": False, "is_general": True, "unlinked": 0, "name_normalized": "general", - "is_shared": False, - "is_ext_shared": False, "is_org_shared": False, "pending_shared": [], "is_pending_ext_shared": False, - "is_member": True, "is_private": False, "is_mpim": False, - "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, - "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, "previous_names": []} + return { + "id": "C061EG9SL", + "name": "general", + "is_channel": True, + "is_group": False, + "is_im": False, + "created": 1449252889, + "creator": "U061F7AUR", + "is_archived": False, + "is_general": True, + "unlinked": 0, + "name_normalized": "general", + "is_shared": False, + "is_ext_shared": False, + "is_org_shared": False, + "pending_shared": [], + "is_pending_ext_shared": False, + "is_member": True, + "is_private": False, + "is_mpim": False, + "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, + "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, + "previous_names": [], + } diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py index dc21220a01391..a1482bd58b32f 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py @@ -23,22 +23,13 @@ def get_stream_by_name(stream_name, config): def test_channel_members_extractor(token_config): response_mock = MagicMock() - response_mock.json.return_value = {"members": [ - "U023BECGF", - "U061F7AUR", - "W012A3CDE" - ]} + response_mock.json.return_value = {"members": ["U023BECGF", "U061F7AUR", "W012A3CDE"]} records = ChannelMembersExtractor(config=token_config, parameters={}, field_path=["members"]).extract_records(response=response_mock) - assert records == [{"member_id": "U023BECGF"}, - {"member_id": "U061F7AUR"}, - {"member_id": "W012A3CDE"}] + assert records == [{"member_id": "U023BECGF"}, {"member_id": "U061F7AUR"}, {"member_id": "W012A3CDE"}] def test_join_channels(token_config, requests_mock, joined_channel): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json={"ok": True, "channel": joined_channel} - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json={"ok": True, "channel": joined_channel}) token = token_config["credentials"]["api_token"] authenticator = TokenAuthenticator(token) channel_filter = token_config["channel_filter"] @@ -51,13 +42,16 @@ def test_join_channels(token_config, requests_mock, joined_channel): def get_channels_retriever_instance(token_config): return ChannelsRetriever( config=token_config, - requester=HttpRequester(name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, - parameters={}), + requester=HttpRequester( + name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, parameters={} + ), record_selector=RecordSelector( extractor=DpathExtractor(field_path=["channels"], config=token_config, parameters={}), - config=token_config, parameters={}, - schema_normalization=None), - parameters={} + config=token_config, + parameters={}, + schema_normalization=None, + ), + parameters={}, ) @@ -76,20 +70,22 @@ def test_join_channels_make_join_channel_slice(token_config): @pytest.mark.parametrize( "join_response, log_message", ( - ({"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, "Successfully joined channel: test channel"), - ({"ok": False, "error": "missing_scope", "needed": "channels:write"}, - "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}"), + ( + {"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, + "Successfully joined channel: test channel", + ), + ( + {"ok": False, "error": "missing_scope", "needed": "channels:write"}, + "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}", + ), ), - ids=["successful_join_to_channel", "failed_join_to_channel"] + ids=["successful_join_to_channel", "failed_join_to_channel"], ) def test_join_channel_read(requests_mock, token_config, joined_channel, caplog, join_response, log_message): - mocked_request = requests_mock.post( - url="https://slack.com/api/conversations.join", - json=join_response - ) + mocked_request = requests_mock.post(url="https://slack.com/api/conversations.join", json=join_response) requests_mock.get( url="https://slack.com/api/conversations.list", - json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]} + json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]}, ) retriever = get_channels_retriever_instance(token_config) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index ae1a589227970..90f6531ca2582 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -17,6 +17,7 @@ def get_stream_by_name(stream_name, config): return stream raise ValueError(f"Stream {stream_name} not found") + @parametrized_configs def test_streams(conversations_list, config, is_valid): source = SourceSlack() diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index b9966eb3594c1..b7a599765bb71 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -36,10 +36,10 @@ def get_stream_by_name(stream_name, config): {}, [ # two messages per each channel - {'channel': 'airbyte-for-beginners', 'ts': 1577866844}, - {'channel': 'airbyte-for-beginners', 'ts': 1577877406}, - {'channel': 'good-reads', 'ts': 1577866844}, - {'channel': 'good-reads', 'ts': 1577877406}, + {"channel": "airbyte-for-beginners", "ts": 1577866844}, + {"channel": "airbyte-for-beginners", "ts": 1577877406}, + {"channel": "good-reads", "ts": 1577866844}, + {"channel": "good-reads", "ts": 1577877406}, ], ), ("2020-01-02T00:00:00Z", "2020-01-01T00:00:00Z", [], {}, [{}]), @@ -55,18 +55,18 @@ def get_stream_by_name(stream_name, config): ), ), ) -def test_threads_stream_slices( - requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result -): +def test_threads_stream_slices(requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result): token_config["channel_filter"] = [] requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", - [{"json": {"messages": messages}}, {"json": {"messages": []}}] + "GET", + "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", + [{"json": {"messages": messages}}, {"json": {"messages": []}}], ) start_date = pendulum.parse(start_date) @@ -76,7 +76,7 @@ def test_threads_stream_slices( authenticator=authenticator, default_start_date=start_date, end_date=end_date, - lookback_window=pendulum.Duration(days=token_config["lookback_window"]) + lookback_window=pendulum.Duration(days=token_config["lookback_window"]), ) slices = list(stream.stream_slices(stream_state=stream_state)) assert slices == expected_result @@ -96,7 +96,7 @@ def test_get_updated_state(authenticator, token_config, current_state, latest_re stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) assert stream._get_updated_state(current_stream_state=current_state, latest_record=latest_record) == expected_state @@ -105,10 +105,10 @@ def test_threads_request_params(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - expected = {'channel': 'airbyte-for-beginners', 'limit': 1000, 'ts': 1577866844} + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + expected = {"channel": "airbyte-for-beginners", "limit": 1000, "ts": 1577866844} assert stream.request_params(stream_slice=threads_slice, stream_state={}) == expected @@ -116,7 +116,7 @@ def test_threads_parse_response(mocker, authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) resp = { "messages": [ @@ -129,14 +129,14 @@ def test_threads_parse_response(mocker, authenticator, token_config): "subscribed": True, "last_read": "1484678597.521003", "unread_count": 0, - "ts": "1482960137.003543" + "ts": "1482960137.003543", } ] } resp_mock = mocker.Mock() resp_mock.json.return_value = resp - threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} - actual_response = list(stream.parse_response(response=resp_mock,stream_slice=threads_slice)) + threads_slice = {"channel": "airbyte-for-beginners", "ts": 1577866844} + actual_response = list(stream.parse_response(response=resp_mock, stream_slice=threads_slice)) assert len(actual_response) == 1 assert actual_response[0]["float_ts"] == 1482960137.003543 assert actual_response[0]["channel_id"] == "airbyte-for-beginners" @@ -147,7 +147,7 @@ def test_backoff(token_config, authenticator, headers, expected_result): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, headers=headers) assert stream.get_backoff_strategy().backoff_time(mocked_response) == expected_result @@ -157,10 +157,7 @@ def test_channels_stream_with_autojoin(authenticator) -> None: """ The test uses the `conversations_list` fixture(autouse=true) as API mocker. """ - expected = [ - {'id': 'airbyte-for-beginners', 'is_member': True}, - {'id': 'good-reads', 'is_member': True} - ] + expected = [{"id": "airbyte-for-beginners", "is_member": True}, {"id": "good-reads", "is_member": True}] stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) assert list(stream.read_records(None)) == expected @@ -169,7 +166,7 @@ def test_next_page_token(authenticator, token_config): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = Mock() mocked_response.json.return_value = {"response_metadata": {"next_cursor": "next page"}} @@ -189,22 +186,24 @@ def test_should_retry(authenticator, token_config, status_code, expected): stream = Threads( authenticator=authenticator, default_start_date=pendulum.parse(token_config["start_date"]), - lookback_window=token_config["lookback_window"] + lookback_window=token_config["lookback_window"], ) mocked_response = MagicMock(spec=Response, status_code=status_code) mocked_response.ok = status_code == 200 assert stream.get_error_handler().interpret_response(mocked_response).response_action == expected + def test_channels_stream_with_include_private_channels_false(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=False, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel' + assert params.get("types") == "public_channel" + def test_channels_stream_with_include_private_channels(authenticator) -> None: stream = Channels(channel_filter=[], include_private_channels=True, authenticator=authenticator) params = stream.request_params(stream_slice={}, stream_state={}) - assert params.get("types") == 'public_channel,private_channel' + assert params.get("types") == "public_channel,private_channel" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py index f41f2efbbbf5c..3e4f299906638 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py @@ -105,12 +105,7 @@ class FullRefreshTest(TestCase): def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -125,12 +120,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -152,12 +142,7 @@ def test_given_two_pages_when_read_then_return_records(self, http_mocker: HttpMo def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_limit(100).build(), @@ -172,12 +157,7 @@ def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -190,12 +170,7 @@ def test_given_http_status_400_when_read_then_stream_did_not_run(self, http_mock def test_given_http_status_401_when_read_then_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -208,12 +183,7 @@ def test_given_http_status_401_when_read_then_config_error(self, http_mocker: Ht def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -229,12 +199,7 @@ def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_m def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -247,12 +212,7 @@ def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_r def test_given_http_status_500_when_read_then_raise_config_error(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) http_mocker.get( _payment_methods_request("parent_id").with_any_query_params().build(), @@ -272,12 +232,7 @@ class IncrementalTest(TestCase): def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None: http_mocker.get( StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), - _customers_response() - .with_record( - _a_customer() - .with_id("parent_id") - ) - .build(), + _customers_response().with_record(_a_customer().with_id("parent_id")).build(), ) cursor_value = int(_A_START_DATE.timestamp()) + 1 http_mocker.get( diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index eaaafcad12ef3..0264c685f6f73 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -523,6 +523,7 @@ def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by ] assert records == [{"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1691629292}] + @freezegun.freeze_time("2023-08-23T15:00:15Z") def test_setup_attempts(requests_mock, incremental_stream_args): requests_mock.get( @@ -583,12 +584,12 @@ def test_setup_attempts(requests_mock, incremental_stream_args): def test_persons_wo_state(requests_mock, stream_args): requests_mock.get("/v1/accounts", json={"data": [{"id": 1, "object": "account", "created": 111}]}) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("full_refresh")) assert slices == [{"parent": {"id": 1, "object": "account", "created": 111}}] requests_mock.get("/v1/accounts/1/persons", json={"data": [{"id": 11, "object": "person", "created": 222}]}) @@ -618,12 +619,12 @@ def test_persons_w_state(requests_mock, stream_args): }, ) stream = UpdatedCursorIncrementalStripeSubStream( - name="persons", - path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", - parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), - event_types=["person.created", "person.updated", "person.deleted"], - **stream_args, - ) + name="persons", + path=lambda self, stream_slice, *args, **kwargs: f"accounts/{stream_slice['parent']['id']}/persons", + parent=StripeStream(name="accounts", path="accounts", use_cache=False, **stream_args), + event_types=["person.created", "person.updated", "person.deleted"], + **stream_args, + ) slices = list(stream.stream_slices("incremental", stream_state={"updated": pendulum.parse("2023-08-20T00:00:00").int_timestamp})) assert slices == [{}] records = [ @@ -802,7 +803,7 @@ def test_subscription_items_extra_request_params(requests_mock, stream_by_name, "created": 1699603175, "quantity": 1, "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", - "subscription_updated": 1699603174, #1699603175 + "subscription_updated": 1699603174, # 1699603175 }, { "id": "si_OynPdzMZykmCWm", diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 77ebfa14f6246..863aa11f2b76e 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -26,9 +26,11 @@ def source_fixture(): @pytest.fixture(name="mock_survey_cto") def mock_survey_cto_fixture(): - with patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, patch( - "source_surveycto.source.Helpers.get_filter_data", return_value="value" - ) as mock_filter_data, patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema: + with ( + patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, + patch("source_surveycto.source.Helpers.get_filter_data", return_value="value") as mock_filter_data, + patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema, + ): yield mock_call_survey_cto, mock_filter_data, mock_json_schema diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py index 74c9ae98ad013..125c7e325a61f 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/conftest.py @@ -11,32 +11,31 @@ from source_surveymonkey.source import SourceSurveymonkey -@pytest.fixture(name='read_json') +@pytest.fixture(name="read_json") def read_json_fixture(request): def read_json(file_name, skip_folder=False): if not skip_folder: - folder_name = request.node.fspath.basename.split('.')[0] + folder_name = request.node.fspath.basename.split(".")[0] with open("unit_tests/" + folder_name + "/" + file_name) as f: return json.load(f) + return read_json -@pytest.fixture(name='read_records') + +@pytest.fixture(name="read_records") def read_records_fixture(config): def read_records(stream_name, slice=StreamSlice(partition={"survey_id": "307785415"}, cursor_slice={})): stream = next(filter(lambda x: x.name == stream_name, SourceSurveymonkey().streams(config=config))) - records = list( - map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) + records = list(map(lambda record: record.data, stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice))) return records + return read_records @pytest.fixture def args_mock(): - return { - "authenticator": None, - "start_date": pendulum.parse("2000-01-01"), - "survey_ids": [] - } + return {"authenticator": None, "start_date": pendulum.parse("2000-01-01"), "survey_ids": []} + @pytest.fixture def config(args_mock): @@ -44,5 +43,5 @@ def config(args_mock): **args_mock, "survey_ids": ["307785415"], "credentials": {"access_token": "access_token"}, - "start_date": args_mock["start_date"].to_iso8601_string() + "start_date": args_mock["start_date"].to_iso8601_string(), } diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py index 24a60bc08d959..42fa21332546a 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -9,10 +9,10 @@ from source_surveymonkey.streams import SurveyIds, Surveys -@pytest.mark.parametrize("stream, expected_records_file, stream_slice", [ - (SurveyIds, "records_survey_ids.json", None), - (Surveys, "records_surveys.json", {"survey_id": "307785415"}) -]) +@pytest.mark.parametrize( + "stream, expected_records_file, stream_slice", + [(SurveyIds, "records_survey_ids.json", None), (Surveys, "records_surveys.json", {"survey_id": "307785415"})], +) def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, expected_records_file, stream_slice): requests_mock.get( "https://api.surveymonkey.com/v3/surveys", @@ -30,8 +30,11 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, } }, }, - {"status_code": 200, "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, - "json": read_json("response_survey_ids.json")}, + { + "status_code": 200, + "headers": {"X-Ratelimit-App-Global-Minute-Remaining": "100"}, + "json": read_json("response_survey_ids.json"), + }, ], ) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) @@ -42,10 +45,10 @@ def test_survey_stream_read_records(requests_mock, args_mock, read_json, stream, assert list(records) == expected_records -@pytest.mark.parametrize("additional_arguments, expected_slices", [ - ({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), - ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}]) -]) +@pytest.mark.parametrize( + "additional_arguments, expected_slices", + [({}, [{"survey_id": "307785415"}, {"survey_id": "307785388"}]), ({"survey_ids": ["307785415"]}, [{"survey_id": "307785415"}])], +) def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments, expected_slices): if not additional_arguments: requests_mock.get("https://api.surveymonkey.com/v3/surveys", json=read_json("response_survey_ids.json")) @@ -54,11 +57,14 @@ def test_survey_slices(requests_mock, args_mock, read_json, additional_arguments assert list(stream_slices) == expected_slices -@pytest.mark.parametrize("endpoint, records_filename", [ - ("survey_pages", "records_survey_pages.json"), - ("survey_questions", "records_survey_questions.json"), - ("survey_collectors", "records_survey_collectors.json") -]) +@pytest.mark.parametrize( + "endpoint, records_filename", + [ + ("survey_pages", "records_survey_pages.json"), + ("survey_questions", "records_survey_questions.json"), + ("survey_collectors", "records_survey_collectors.json"), + ], +) def test_survey_data(requests_mock, read_records, read_json, endpoint, records_filename): requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/details", json=read_json("response_survey_details.json")) requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/collectors", json=read_json("response_survey_collectors.json")) diff --git a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py index 235d12e640af0..1e2ca8425a3f1 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/unit_tests/test_paginator.py @@ -9,60 +9,57 @@ def create_response(current_page: int, total_pages: int) -> requests.Response: """Helper function to create mock responses""" response = MagicMock(spec=requests.Response) - response.json.return_value = { - "response": { - "currentPage": current_page, - "pages": total_pages - } - } + response.json.return_value = {"response": {"currentPage": current_page, "pages": total_pages}} return response + @pytest.mark.parametrize( "current_page,total_pages,expected_next_page", [ - (1, 5, 2), # First page - (2, 5, 3), # Middle page - (4, 5, 5), # Second to last page - (5, 5, None), # Last page - (1, 1, None), # Single page + (1, 5, 2), # First page + (2, 5, 3), # Middle page + (4, 5, 5), # Second to last page + (5, 5, None), # Last page + (1, 1, None), # Single page ], - ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"] + ids=["First page", "Middle page", "Penultimate page", "Last page", "Single page"], ) def test_page_increment(connector_dir, components_module, current_page, total_pages, expected_next_page): """Test the CustomPageIncrement pagination for various page combinations""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Set internal page counter to match current_page paginator._page = current_page - + mock_response = create_response(current_page, total_pages) next_page = paginator.next_page_token(mock_response) assert next_page == expected_next_page, f"Page {current_page} of {total_pages} should get next_page={expected_next_page}" + def test_reset_functionality(components_module): """Test the reset behavior of CustomPageIncrement""" CustomPageIncrement = components_module.CustomPageIncrement - + config = {} page_size = 10 parameters = {} paginator = CustomPageIncrement(config, page_size, parameters) - + # Advance a few pages mock_response = create_response(current_page=1, total_pages=5) paginator.next_page_token(mock_response) paginator.next_page_token(create_response(current_page=2, total_pages=5)) - + # Test reset paginator.reset() assert paginator._page == 1, "Reset should set page back to 1" - + # Verify pagination works after reset next_page = paginator.next_page_token(mock_response) assert next_page == 2, "Should increment to page 2 after reset" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py index 628a57a7a89fe..21b5ae6ecceaa 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/components/semi_incremental_record_filter.py @@ -7,7 +7,6 @@ class PerPartitionRecordFilter(RecordFilter): - """ Prepares per partition stream state to be used in the Record Filter condition. Gets current stream state cursor value for stream slice and passes it to condition. diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py index 5c32c0b9ea7d8..c01481cb119e5 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/config_builder.py @@ -7,14 +7,9 @@ class ConfigBuilder: def __init__(self) -> None: self._config: Dict[str, Any] = { - "credentials": { - "auth_type": "oauth2.0", - "access_token": "access token", - "app_id": "11111111111111111111", - "secret": "secret" - }, + "credentials": {"auth_type": "oauth2.0", "access_token": "access token", "app_id": "11111111111111111111", "secret": "secret"}, "start_date": "2024-01-01", - "include_deleted": False + "include_deleted": False, } def with_include_deleted(self) -> "ConfigBuilder": diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py index b0a8f102ce3dc..64b45155ed502 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/integration/test_reports_hourly.py @@ -88,10 +88,11 @@ class TestAdsReportHourly(TestCase): "real_time_app_install_cost", "app_install", ] + def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -113,16 +114,16 @@ def state(self): def mock_response(self, http_mocker: HttpMocker, include_deleted=False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_AD", - "dimensions": '["ad_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_AD", + "dimensions": '["ad_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "ad_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( @@ -132,7 +133,7 @@ def mock_response(self, http_mocker: HttpMocker, include_deleted=False): ), HttpResponse(body=json.dumps(find_template(self.stream_name, __file__)), status_code=200), ) - query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] + query_params["start_date"] = query_params["end_date"] = self.config()["end_date"] http_mocker.get( HttpRequest( @@ -239,7 +240,7 @@ class TestAdGroupsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -263,16 +264,16 @@ def state(self): def test_basic_read(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -348,17 +349,17 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): mock_advertisers_slices(http_mocker, self.config()) filtering = '[{"field_name": "adgroup_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_ADGROUP", - "dimensions": '["adgroup_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - "filtering": filtering, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_ADGROUP", + "dimensions": '["adgroup_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + "filtering": filtering, + } http_mocker.get( HttpRequest( url=f"https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/", @@ -380,6 +381,7 @@ def test_read_with_include_deleted(self, http_mocker: HttpMocker): assert output.records[0].record.data.get("adgroup_id") is not None assert output.records[0].record.data.get("stat_time_hour") is not None + class TestAdvertisersReportsHourly(TestCase): stream_name = "advertisers_reports_hourly" advertiser_id = "872746382648" @@ -536,7 +538,7 @@ class TestCampaignsReportsHourly(TestCase): def catalog(self, sync_mode: SyncMode = SyncMode.full_refresh): return CatalogBuilder().with_stream(name=self.stream_name, sync_mode=sync_mode).build() - def config(self, include_deleted:bool=False): + def config(self, include_deleted: bool = False): config_to_build = ConfigBuilder().with_end_date("2024-01-02") if include_deleted: config_to_build = config_to_build.with_include_deleted() @@ -556,18 +558,18 @@ def state(self): .build() ) - def mock_response(self, http_mocker: HttpMocker, include_deleted:bool=False): + def mock_response(self, http_mocker: HttpMocker, include_deleted: bool = False): query_params = { - "service_type": "AUCTION", - "report_type": "BASIC", - "data_level": "AUCTION_CAMPAIGN", - "dimensions": '["campaign_id", "stat_time_hour"]', - "metrics": str(self.metrics).replace("'", '"'), - "start_date": self.config()["start_date"], - "end_date": self.config()["start_date"], - "page_size": 1000, - "advertiser_id": self.advertiser_id, - } + "service_type": "AUCTION", + "report_type": "BASIC", + "data_level": "AUCTION_CAMPAIGN", + "dimensions": '["campaign_id", "stat_time_hour"]', + "metrics": str(self.metrics).replace("'", '"'), + "start_date": self.config()["start_date"], + "end_date": self.config()["start_date"], + "page_size": 1000, + "advertiser_id": self.advertiser_id, + } if include_deleted: query_params["filtering"] = '[{"field_name": "campaign_status", "filter_type": "IN", "filter_value": "[\\"STATUS_ALL\\"]"}]' http_mocker.get( diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py index 1825fa49d955b..7d5a6889e0ea9 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_components.py @@ -21,7 +21,7 @@ [ ({"credentials": {"advertiser_id": "11111111111"}}, "11111111111"), ({"environment": {"advertiser_id": "2222222222"}}, "2222222222"), - ({"credentials": {"access_token": "access_token"}}, None) + ({"credentials": {"access_token": "access_token"}}, None), ], ) def test_get_partition_value_from_config(config, expected): @@ -29,8 +29,9 @@ def test_get_partition_value_from_config(config, expected): parent_stream_configs=[MagicMock()], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) actual = router.get_partition_value_from_config() assert actual == expected @@ -39,17 +40,26 @@ def test_get_partition_value_from_config(config, expected): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, - [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_ids": '["11111111111"]', "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_ids": '["2222222222"]', "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], - {"code": 0, "message": "ok", "data": - {"list": [{"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_ids": '["11111111", "22222222"]', "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_multiple(config, expected, requests_mock, json_data): @@ -57,23 +67,19 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = MultipleAdvertiserIdsPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_ids", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_ids", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_ids" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_ids", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -81,19 +87,26 @@ def test_stream_slices_multiple(config, expected, requests_mock, json_data): @pytest.mark.parametrize( "config, expected, json_data", [ - ({"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, [{"advertiser_id": "11111111111", "parent_slice": {}}], - None), + ( + {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}}, + [{"advertiser_id": "11111111111", "parent_slice": {}}], + None, + ), ({"environment": {"advertiser_id": "2222222222"}}, [{"advertiser_id": "2222222222", "parent_slice": {}}], None), ( - {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, - [{"advertiser_id": "11111111", "parent_slice": {}}, - {"advertiser_id": "22222222", "parent_slice": {}}], - {"code": 0, "message": "ok", - "data": {"list": [ - {"advertiser_id": "11111111", "advertiser_name": "name"}, - {"advertiser_id": "22222222", "advertiser_name": "name"}]} - } - ) + {"credentials": {"auth_type": "oauth2.0", "access_token": "access_token"}}, + [{"advertiser_id": "11111111", "parent_slice": {}}, {"advertiser_id": "22222222", "parent_slice": {}}], + { + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "11111111", "advertiser_name": "name"}, + {"advertiser_id": "22222222", "advertiser_name": "name"}, + ] + }, + }, + ), ], ) def test_stream_slices_single(config, expected, requests_mock, json_data): @@ -101,23 +114,19 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): advertiser_ids_stream = advertiser_ids_stream[0] if advertiser_ids_stream else MagicMock() router = SingleAdvertiserIdPerPartition( - parent_stream_configs=[ParentStreamConfig( - partition_field="advertiser_id", - config=config, - parent_key="advertiser_id", - stream=advertiser_ids_stream, - parameters={} - )], + parent_stream_configs=[ + ParentStreamConfig( + partition_field="advertiser_id", config=config, parent_key="advertiser_id", stream=advertiser_ids_stream, parameters={} + ) + ], config=config, parameters={ - "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], "partition_field": "advertiser_id" - } + "path_in_config": [["credentials", "advertiser_id"], ["environment", "advertiser_id"]], + "partition_field": "advertiser_id", + }, ) if json_data: - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_data - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_data) actual = list(router.stream_slices()) assert actual == expected @@ -126,22 +135,22 @@ def test_stream_slices_single(config, expected, requests_mock, json_data): "records, state, slice, expected", [ ( - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], - {}, - {}, - [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}] + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], + {}, + {}, + [{"id": 1, "start_time": "2024-01-01"}, {"id": 2, "start_time": "2024-01-01"}], ), ( - [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 1}, - [{"advertiser_id": 1, "start_time": "2024-01-02"}] + [{"advertiser_id": 1, "start_time": "2022-01-01"}, {"advertiser_id": 1, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 1}, + [{"advertiser_id": 1, "start_time": "2024-01-02"}], ), ( - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], - {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, - {"advertiser_id": 2}, - [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], + {"states": [{"partition": {"advertiser_id": 1, "parent_slice": {}}, "cursor": {"start_time": "2023-12-31"}}]}, + {"advertiser_id": 2}, + [{"advertiser_id": 2, "start_time": "2022-01-01"}, {"advertiser_id": 2, "start_time": "2024-01-02"}], ), ], ) @@ -150,18 +159,20 @@ def test_record_filter(records, state, slice, expected): record_filter = PerPartitionRecordFilter( config=config, parameters={"partition_field": "advertiser_id"}, - condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}" + condition="{{ record['start_time'] >= stream_state.get('start_time', config.get('start_date', '')) }}", + ) + filtered_records = list( + record_filter.filter_records(records=records, stream_state=state, stream_slice=StreamSlice(partition=slice, cursor_slice={})) ) - filtered_records = list(record_filter.filter_records( - records=records, - stream_state=state, - stream_slice=StreamSlice(partition=slice, cursor_slice={}) - )) assert filtered_records == expected def test_hourly_datetime_based_cursor(): - config = {"credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, "start_date": "2022-01-01", "end_date": "2022-01-02"} + config = { + "credentials": {"auth_type": "oauth2.0", "advertiser_id": "11111111111"}, + "start_date": "2022-01-01", + "end_date": "2022-01-02", + } cursor = HourlyDatetimeBasedCursor( start_datetime=MinMaxDatetime(datetime="{{ config.get('start_date', '2016-09-01') }}", datetime_format="%Y-%m-%d", parameters={}), @@ -172,42 +183,33 @@ def test_hourly_datetime_based_cursor(): cursor_field="stat_time_hour", datetime_format="%Y-%m-%d", cursor_datetime_formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"], - parameters={} + parameters={}, ) cursor._cursor = "2022-01-01 00:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] cursor._cursor = "2022-01-01 10:00:00" partition_daterange = list(cursor.stream_slices()) assert partition_daterange == [ {"start_time": "2022-01-01", "end_time": "2022-01-01"}, - {"start_time": "2022-01-02", "end_time": "2022-01-02"} + {"start_time": "2022-01-02", "end_time": "2022-01-02"}, ] @pytest.mark.parametrize( "record, expected", [ + ({"metrics": {"metric_1": "not empty", "metric_2": "-"}}, {"metrics": {"metric_1": "not empty", "metric_2": None}}), + ({"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}), ( - {"metrics": {"metric_1": "not empty", "metric_2": "-"}}, - {"metrics": {"metric_1": "not empty", "metric_2": None}} - ), - ( - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}}, - {"metrics": {"metric_1": "not empty", "metric_2": "not empty"}} - ), - ( - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, - {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}} - ), - ( - {}, - {} + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, + {"dimensions": {"dimension_1": "not empty", "dimension_2": "not empty"}}, ), + ({}, {}), ], ) def test_transform_empty_metrics(record, expected): diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py index 44db480d1000d..77b58d31b283b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/test_source.py @@ -15,8 +15,24 @@ [ ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01"}, 36), ({"access_token": "token", "start_date": "2021-01-01", "environment": {"advertiser_id": "1111"}}, 28), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "LIFETIME"}, 15), - ({"access_token": "token", "environment": {"app_id": "1111", "secret": "secret"}, "start_date": "2021-04-01", "report_granularity": "DAY"}, 27), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "LIFETIME", + }, + 15, + ), + ( + { + "access_token": "token", + "environment": {"app_id": "1111", "secret": "secret"}, + "start_date": "2021-04-01", + "report_granularity": "DAY", + }, + 27, + ), ], ) def test_source_streams(config, stream_len): @@ -43,11 +59,27 @@ def config_fixture(): def test_source_check_connection_ok(config, requests_mock): requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) requests_mock.get( "https://business-api.tiktok.com/open_api/v1.3/advertiser/info/?page_size=100&advertiser_ids=%5B%22917429327%22%5D", - json={"code": 0, "message": "ok", "data": {"list": [{"advertiser_id": "917429327", "advertiser_name": "name"}, ]}} + json={ + "code": 0, + "message": "ok", + "data": { + "list": [ + {"advertiser_id": "917429327", "advertiser_name": "name"}, + ] + }, + }, ) logger_mock = MagicMock() assert SourceTiktokMarketing().check_connection(logger_mock, config) == (True, None) @@ -56,20 +88,17 @@ def test_source_check_connection_ok(config, requests_mock): @pytest.mark.parametrize( "json_response, expected_result, expected_message", [ - ({"code": 40105, "message": "Access token is incorrect or has been revoked."}, - (False, "Access token is incorrect or has been revoked."), - None), - ({"code": 40100, "message": "App reaches the QPS limit."}, - None, - 38) - ] + ( + {"code": 40105, "message": "Access token is incorrect or has been revoked."}, + (False, "Access token is incorrect or has been revoked."), + None, + ), + ({"code": 40100, "message": "App reaches the QPS limit."}, None, 38), + ], ) @pytest.mark.usefixtures("mock_sleep") def test_source_check_connection_failed(config, requests_mock, capsys, json_response, expected_result, expected_message): - requests_mock.get( - "https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", - json=json_response - ) + requests_mock.get("https://business-api.tiktok.com/open_api/v1.3/oauth2/advertiser/get/", json=json_response) logger_mock = MagicMock() result = SourceTiktokMarketing().check_connection(logger_mock, config) diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py index 309355222b869..17c3f3e577c6b 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py @@ -60,14 +60,14 @@ def test_data_field(self, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, expected", [ - (Accounts, ['name']), + (Accounts, ["name"]), ], ) def test_changeable_fields(self, stream_cls, expected): - with patch.object(Accounts, "changeable_fields", ['name']): - stream = stream_cls(**self.CONFIG) - result = stream.changeable_fields - assert result == expected + with patch.object(Accounts, "changeable_fields", ["name"]): + stream = stream_cls(**self.CONFIG) + result = stream.changeable_fields + assert result == expected @pytest.mark.parametrize( "stream_cls, expected", @@ -108,12 +108,12 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): with patch.object(TwilioStream, "changeable_fields", ["name"]): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = list(stream.parse_response(response)) - assert result[0]['id'] == expected[0]['id'] + stream = stream_cls(**self.CONFIG) + url = f"{stream.url_base}{stream.path()}" + requests_mock.get(url, json=test_response) + response = requests.get(url) + result = list(stream.parse_response(response)) + assert result[0]["id"] == expected[0]["id"] @pytest.mark.parametrize( "stream_cls, expected", @@ -151,7 +151,7 @@ def test_request_params(self, stream_cls, next_page_token, expected): ("Fri, 11 Dec 2020 04:28:40 +0000", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("2020-12-11T04:28:40Z", {"format": "date-time"}, "2020-12-11T04:28:40Z"), ("some_string", {}, "some_string"), - ] + ], ) def test_transform_function(self, original_value, field_schema, expected_value): assert Accounts.custom_transform_function(original_value, field_schema) == expected_value diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py index a841b1d266cff..5f8ccc34befbb 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py @@ -7,7 +7,9 @@ def test_typeform_authenticator(): config = {"credentials": {"auth_type": "access_token", "access_token": "access_token"}} - oauth_config = {"credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} + oauth_config = { + "credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"} + } class TokenProvider: def get_token(self) -> str: @@ -16,13 +18,13 @@ def get_token(self) -> str: auth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(auth, BearerAuthenticator) oauth = TypeformAuthenticator( token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), config=oauth_config, - oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token"), ) assert isinstance(oauth, DeclarativeSingleUseRefreshTokenOauth2Authenticator) diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py index f7e3daa64937d..d8eea511db56d 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py @@ -256,7 +256,6 @@ def get_json_schema(self) -> Mapping[str, Any]: class SourceWebflow(AbstractSource): - """This is the main class that defines the methods that will be called by Airbyte infrastructure""" @staticmethod diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py index ea40dc0ab320c..4b60619846493 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py @@ -4,7 +4,6 @@ class WebflowToAirbyteMapping: - """ The following disctionary is used for dynamically pulling the schema from Webflow, and mapping it to an Airbyte-compatible json-schema Webflow: https://developers.webflow.com/#get-collection-with-full-schema diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py index 13e3b37c9f128..12a745943f2fa 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_custom_parsing.py @@ -10,7 +10,9 @@ def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -22,7 +24,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py index ee4687b936ef2..3765b99482f84 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py @@ -22,6 +22,7 @@ def test_check_connection_failed(bad_config, requests_mock): assert check_succeeded is False assert error == "" or "none" in error.lower() + def test_streams_count(config_pass): source = SourceXero() streams = source.streams(config_pass) diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py index a0bdf0e116109..1a40d923f1ae2 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py @@ -10,7 +10,9 @@ def test_parsed_result(requests_mock, config_pass, mock_bank_transaction_response): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"]) + requests_mock.get( + url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=200, json=mock_bank_transaction_response["BankTransactions"] + ) stream = get_stream_by_name("bank_transactions", config_pass) expected_record = mock_bank_transaction_response["BankTransactions"] for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): @@ -26,8 +28,8 @@ def test_request_params(config_pass): def test_request_headers(config_pass): bank_transactions = get_stream_by_name("bank_transactions", config_pass) - expected_headers = {'Xero-Tenant-Id': 'goodone', 'Accept': 'application/json'} - assert bank_transactions.retriever.requester.get_request_headers() == expected_headers + expected_headers = {"Xero-Tenant-Id": "goodone", "Accept": "application/json"} + assert bank_transactions.retriever.requester.get_request_headers() == expected_headers def test_http_method(config_pass): @@ -38,7 +40,7 @@ def test_http_method(config_pass): def test_ignore_forbidden(requests_mock, config_pass): - requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{ "message": "Forbidden resource"}]) + requests_mock.get(url="https://api.xero.com/api.xro/2.0/BankTransactions", status_code=403, json=[{"message": "Forbidden resource"}]) stream = get_stream_by_name("bank_transactions", config_pass) records = [] @@ -52,7 +54,9 @@ def test_parse_date(): # 11/10/2020 00:00:00 +3 (11/10/2020 21:00:00 GMT/UTC) assert ParseDates.parse_date("/Date(1602363600000+0300)/") == datetime.datetime(2020, 10, 11, 0, 0, tzinfo=datetime.timezone.utc) # 02/02/2020 10:31:51.5 +3 (02/02/2020 07:31:51.5 GMT/UTC) - assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime(2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc) + assert ParseDates.parse_date("/Date(1580628711500+0300)/") == datetime.datetime( + 2020, 2, 2, 10, 31, 51, 500000, tzinfo=datetime.timezone.utc + ) # 07/02/2022 20:12:55 GMT/UTC assert ParseDates.parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py index 1addd15641563..7848076a6b1ac 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py @@ -87,9 +87,9 @@ def collect_cursor_values(self) -> Mapping[str, Optional[int]]: """ cursor_values: dict = { "state": self._cursor if self._cursor else self._start_boundary, - "highest_observed_record_value": self._highest_observed_record_cursor_value - if self._highest_observed_record_cursor_value - else self._start_boundary, + "highest_observed_record_value": ( + self._highest_observed_record_cursor_value if self._highest_observed_record_cursor_value else self._start_boundary + ), } # filter out the `NONE` STATE values from the `cursor_values` return {key: value for key, value in cursor_values.items()} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py index c48196cfa1edd..8083a1cc5e0d3 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py @@ -12,25 +12,14 @@ def config() -> Mapping[str, Any]: return { "start_date": "2020-10-01T00:00:00Z", "subdomain": "", - "credentials": { - "credentials": "access_token", - "access_token": "__access_token__" - } + "credentials": {"credentials": "access_token", "access_token": "__access_token__"}, } @pytest.fixture def bans_stream_record() -> Mapping[str, Any]: return { - "ip_address": [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - } - ], + "ip_address": [{"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}], "visitor": [ { "type": "visitor", @@ -38,28 +27,22 @@ def bans_stream_record() -> Mapping[str, Any]: "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", } - ] + ], } @pytest.fixture def bans_stream_record_extractor_expected_output() -> List[Mapping[str, Any]]: return [ - { - "reason": "test", - "type": "ip_address", - "id": 1234, - "created_at": "2021-04-21T14:42:46Z", - "ip_address": "0.0.0.0" - }, + {"reason": "test", "type": "ip_address", "id": 1234, "created_at": "2021-04-21T14:42:46Z", "ip_address": "0.0.0.0"}, { "type": "visitor", "id": 4444, "visitor_name": "Visitor 4444", "visitor_id": "visitor_id", "reason": "test", - "created_at": "2021-04-27T13:25:01Z" + "created_at": "2021-04-27T13:25:01Z", }, ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py index 60ae75a17da15..d33b2302e1616 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py @@ -8,8 +8,8 @@ def test_bans_stream_record_extractor( config, - requests_mock, - bans_stream_record, + requests_mock, + bans_stream_record, bans_stream_record_extractor_expected_output, ) -> None: test_url = f"https://{config['subdomain']}.zendesk.com/api/v2/chat/bans" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py index 9557a312b6359..75dad4a8f0d54 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -9,26 +9,24 @@ def _get_cursor(config) -> ZendeskChatIdIncrementalCursor: return ZendeskChatIdIncrementalCursor( - config = config, - cursor_field = "id", - field_name = "since_id", - parameters = {}, + config=config, + cursor_field="id", + field_name="since_id", + parameters={}, ) @pytest.mark.parametrize( "stream_state, expected_cursor_value, expected_state_value", [ - ({"id": 10}, 10, {'id': 10}), + ({"id": 10}, 10, {"id": 10}), ], - ids=[ - "SET Initial State and GET State" - ] + ids=["SET Initial State and GET State"], ) def test_id_incremental_cursor_set_initial_state_and_get_stream_state( - config, + config, stream_state, - expected_cursor_value, + expected_cursor_value, expected_state_value, ) -> None: cursor = _get_cursor(config) @@ -44,17 +42,14 @@ def test_id_incremental_cursor_set_initial_state_and_get_stream_state( ({"id": 123}, 123), ({"id": 456}, 456), ], - ids=[ - "first", - "second" - ] + ids=["first", "second"], ) def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: cursor = _get_cursor(config) cursor.observe(stream_slice={}, record=test_record) cursor.close_slice(stream_slice={}) assert cursor._cursor == expected - + @pytest.mark.parametrize( "stream_state, input_slice, expected", @@ -62,17 +57,14 @@ def test_id_incremental_cursor_close_slice(config, test_record, expected) -> Non ({}, {"id": 1}, {}), ({"id": 2}, {"id": 1}, {"since_id": 2}), ], - ids=[ - "No State", - "With State" - ] + ids=["No State", "With State"], ) def test_id_incremental_cursor_get_request_params(config, stream_state, input_slice, expected) -> None: cursor = _get_cursor(config) if stream_state: cursor.set_initial_state(stream_state) assert cursor.get_request_params(stream_slice=input_slice) == expected - + @pytest.mark.parametrize( "stream_state, record, expected", @@ -85,7 +77,7 @@ def test_id_incremental_cursor_get_request_params(config, stream_state, input_sl "No State", "With State > Record value", "With State < Record value", - ] + ], ) def test_id_incremental_cursor_should_be_synced(config, stream_state, record, expected) -> None: cursor = _get_cursor(config) @@ -107,7 +99,7 @@ def test_id_incremental_cursor_should_be_synced(config, stream_state, record, ex "First < Second - should not be synced", "Has First but no Second - should be synced", "Has no First and has no Second - should not be synced", - ] + ], ) def test_id_incremental_cursor_is_greater_than_or_equal(config, first_record, second_record, expected) -> None: cursor = _get_cursor(config) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py index ddf84932adc0d..48f06329adbd8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py @@ -10,19 +10,16 @@ def _get_paginator(config, id_field) -> ZendeskChatIdOffsetIncrementPaginationStrategy: return ZendeskChatIdOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - id_field = id_field, - parameters = {}, + config=config, + page_size=1, + id_field=id_field, + parameters={}, ) @pytest.mark.parametrize( "id_field, last_records, expected", - [ - ("id", [{"id": 1}], 2), - ("id", [], None) - ], + [("id", [{"id": 1}], 2), ("id", [], None)], ) def test_id_offset_increment_pagination_next_page_token(requests_mock, config, id_field, last_records, expected) -> None: paginator = _get_paginator(config, id_field) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py index 61d793cbc9f96..10838351b5c5d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py @@ -10,18 +10,18 @@ def _get_paginator(config, time_field_name) -> ZendeskChatTimeOffsetIncrementPaginationStrategy: return ZendeskChatTimeOffsetIncrementPaginationStrategy( - config = config, - page_size = 1, - time_field_name = time_field_name, - parameters = {}, + config=config, + page_size=1, + time_field_name=time_field_name, + parameters={}, ) @pytest.mark.parametrize( "time_field_name, response, last_records, expected", [ - ("end_time", {"chats":[{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), - ("end_time", {"chats":[], "end_time": 3}, [], None), + ("end_time", {"chats": [{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), + ("end_time", {"chats": [], "end_time": 3}, [], None), ], ) def test_time_offset_increment_pagination_next_page_token(requests_mock, config, time_field_name, response, last_records, expected) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py index a98cc8283e930..0fdf55c8c469c 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -10,17 +10,17 @@ def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampCursor: cursor = ZendeskChatTimestampCursor( - start_datetime = "2020-10-01T00:00:00Z", - cursor_field = cursor_field, - datetime_format = "%s", - config = config, - parameters = {}, - use_microseconds = f"{{{ {use_microseconds} }}}", + start_datetime="2020-10-01T00:00:00Z", + cursor_field=cursor_field, + datetime_format="%s", + config=config, + parameters={}, + use_microseconds=f"{{{ {use_microseconds} }}}", ) # patching missing parts cursor.start_time_option = RequestOption( - field_name = cursor_field, - inject_into = RequestOptionType.request_parameter, + field_name=cursor_field, + inject_into=RequestOptionType.request_parameter, parameters={}, ) return cursor @@ -29,25 +29,25 @@ def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampC @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), + (True, {"start_time": 1}, {"start_time": 1000000}), ], ) def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) test_result = cursor.add_microseconds({}, input_slice) assert test_result == expected - + @pytest.mark.parametrize( "use_microseconds, input_slice, expected", [ - (True, {"start_time": 1}, {'start_time': 1000000}), - (False, {"start_time": 1}, {'start_time': 1}), + (True, {"start_time": 1}, {"start_time": 1000000}), + (False, {"start_time": 1}, {"start_time": 1}), ], ids=[ "WITH `use_microseconds`", "WITHOUT `use_microseconds`", - ] + ], ) def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py index 5f02c2b74fbbf..c384843976a70 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/helpers.py @@ -81,30 +81,28 @@ def given_post_comments( ) return post_comments_record_builder + def given_tickets(http_mocker: HttpMocker, start_date: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: """ Tickets requests setup """ - tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field( - FieldPath("generated_timestamp"), start_date.int_timestamp - ) + tickets_record_builder = TicketsRecordBuilder.tickets_record().with_field(FieldPath("generated_timestamp"), start_date.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder -def given_tickets_with_state(http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator) -> TicketsRecordBuilder: + +def given_tickets_with_state( + http_mocker: HttpMocker, start_date: DateTime, cursor_value: DateTime, api_token_authenticator: ApiTokenAuthenticator +) -> TicketsRecordBuilder: """ Tickets requests setup """ tickets_record_builder = TicketsRecordBuilder.tickets_record().with_cursor(cursor_value.int_timestamp) http_mocker.get( - TicketsRequestBuilder.tickets_endpoint(api_token_authenticator) - .with_start_time(start_date.int_timestamp) - .build(), + TicketsRequestBuilder.tickets_endpoint(api_token_authenticator).with_start_time(start_date.int_timestamp).build(), TicketsResponseBuilder.tickets_response().with_record(tickets_record_builder).build(), ) return tickets_record_builder @@ -114,24 +112,20 @@ def given_groups_with_later_records( http_mocker: HttpMocker, updated_at_value: DateTime, later_record_time_delta: pendulum.duration, - api_token_authenticator: ApiTokenAuthenticator + api_token_authenticator: ApiTokenAuthenticator, ) -> GroupsRecordBuilder: """ Creates two group records one with a specific cursor value and one that has a later cursor value based on the provided timedelta. This is intended to create multiple records with different times which can be used to test functionality like semi-incremental record filtering """ - groups_record_builder = GroupsRecordBuilder.groups_record().with_field( - FieldPath("updated_at"), datetime_to_string(updated_at_value) - ) + groups_record_builder = GroupsRecordBuilder.groups_record().with_field(FieldPath("updated_at"), datetime_to_string(updated_at_value)) later_groups_record_builder = GroupsRecordBuilder.groups_record().with_field( FieldPath("updated_at"), datetime_to_string(updated_at_value + later_record_time_delta) ) http_mocker.get( - GroupsRequestBuilder.groups_endpoint(api_token_authenticator) - .with_page_size(100) - .build(), + GroupsRequestBuilder.groups_endpoint(api_token_authenticator).with_page_size(100).build(), GroupsResponseBuilder.groups_response().with_record(groups_record_builder).with_record(later_groups_record_builder).build(), ) return groups_record_builder diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py index 032b9ed1d3535..34d049d265b8d 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_groups.py @@ -62,9 +62,7 @@ def test_given_incoming_state_semi_incremental_groups_does_not_emit_earlier_reco api_token_authenticator, ) - state_value = { - "updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50)) - } + state_value = {"updated_at": datetime_to_string(pendulum.now(tz="UTC").subtract(years=1, weeks=50))} state = StateBuilder().with_stream_state("groups", state_value).build() diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py index ffe141cdfcb4f..a6d76d6e6b4e4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comment_votes.py @@ -96,7 +96,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -126,7 +132,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py index a9d2d825c5bb0..7c567010d8378 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_comments.py @@ -84,7 +84,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +115,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py index a2617b5a07df1..6069a11eb71ac 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_post_votes.py @@ -84,7 +84,13 @@ def test_given_403_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Forbidden. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mocker): @@ -109,7 +115,13 @@ def test_given_404_error_when_read_posts_comments_then_skip_stream(self, http_mo assert len(output.records) == 0 info_logs = get_log_messages_by_log_level(output.logs, LogLevel.INFO) - assert any(["Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." in error for error in info_logs]) + assert any( + [ + "Not found. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + in error + for error in info_logs + ] + ) @HttpMocker() def test_given_500_error_when_read_posts_comments_then_stop_syncing(self, http_mocker): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py index 699d09742f5df..841c736d04607 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_ticket_metrics.py @@ -20,6 +20,7 @@ _NOW = pendulum.now(tz="UTC") _TWO_YEARS_AGO_DATETIME = _NOW.subtract(years=2) + @freezegun.freeze_time(_NOW.isoformat()) class TestTicketMetricsIncremental(TestCase): @@ -27,10 +28,10 @@ class TestTicketMetricsIncremental(TestCase): def _config(self): return ( ConfigBuilder() - .with_basic_auth_credentials("user@example.com", "password") - .with_subdomain("d3v-airbyte") - .with_start_date(_TWO_YEARS_AGO_DATETIME) - .build() + .with_basic_auth_credentials("user@example.com", "password") + .with_subdomain("d3v-airbyte") + .with_start_date(_TWO_YEARS_AGO_DATETIME) + .build() ) def _get_authenticator(self, config): @@ -46,7 +47,7 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec http_mocker.get( TicketMetricsRequestBuilder.stateless_ticket_metrics_endpoint(api_token_authenticator).with_page_size(100).build(), - TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build() + TicketMetricsResponseBuilder.stateless_ticket_metrics_response().with_record(ticket_metrics_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) @@ -55,7 +56,6 @@ def test_given_no_state_and_successful_sync_when_read_then_set_state_to_most_rec assert output.most_recent_state.stream_descriptor.name == "ticket_metrics" assert output.most_recent_state.stream_state.__dict__ == {"_ab_updated_at": pendulum.parse(record_updated_at).int_timestamp} - @HttpMocker() def test_given_state_and_successful_sync_when_read_then_return_record(self, http_mocker): api_token_authenticator = self._get_authenticator(self._config) @@ -63,16 +63,20 @@ def test_given_state_and_successful_sync_when_read_then_return_record(self, http state_cursor_value = pendulum.now(tz="UTC").subtract(days=2).int_timestamp state = StateBuilder().with_stream_state("ticket_metrics", state={"_ab_updated_at": state_cursor_value}).build() record_cursor_value = pendulum.now(tz="UTC").subtract(days=1) - tickets_records_builder = given_tickets_with_state(http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value,api_token_authenticator) + tickets_records_builder = given_tickets_with_state( + http_mocker, pendulum.from_timestamp(state_cursor_value), record_cursor_value, api_token_authenticator + ) ticket = tickets_records_builder.build() - ticket_metrics_first_record_builder = TicketMetricsRecordBuilder.stateful_ticket_metrics_record().with_field( - FieldPath("ticket_id"), ticket["id"] - ).with_cursor(ticket["generated_timestamp"]) + ticket_metrics_first_record_builder = ( + TicketMetricsRecordBuilder.stateful_ticket_metrics_record() + .with_field(FieldPath("ticket_id"), ticket["id"]) + .with_cursor(ticket["generated_timestamp"]) + ) http_mocker.get( TicketMetricsRequestBuilder.stateful_ticket_metrics_endpoint(api_token_authenticator, ticket["id"]).build(), - TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build() + TicketMetricsResponseBuilder.stateful_ticket_metrics_response().with_record(ticket_metrics_first_record_builder).build(), ) output = read_stream("ticket_metrics", SyncMode.incremental, self._config, state) diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py index a185dd225a2f1..8d72e988f203e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/base_request_builder.py @@ -30,12 +30,7 @@ def request_body(self) -> Optional[str]: """A request body""" def build(self) -> HttpRequest: - return HttpRequest( - url=self.url, - query_params=self.query_params, - headers=self.headers, - body=self.request_body - ) + return HttpRequest(url=self.url, query_params=self.query_params, headers=self.headers, body=self.request_body) class ZendeskSupportBaseRequestBuilder(ZendeskSuppportRequestBuilder): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py index 3d224cd794564..94629bb155266 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/zs_requests/post_comment_votes_request_builder.py @@ -10,7 +10,9 @@ class PostCommentVotesRequestBuilder(ZendeskSupportBaseRequestBuilder): @classmethod - def post_comment_votes_endpoint(cls, authenticator: Authenticator, post_id: int, post_comment_id: int) -> "PostCommentVotesRequestBuilder": + def post_comment_votes_endpoint( + cls, authenticator: Authenticator, post_id: int, post_comment_id: int + ) -> "PostCommentVotesRequestBuilder": return cls("d3v-airbyte", f"community/posts/{post_id}/comments/{post_comment_id}/votes").with_authenticator(authenticator) def __init__(self, subdomain: str, resource: str) -> None: diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index ae922ca7ffc81..481abff11379c 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -143,7 +143,9 @@ def test_convert_config2stream_args(config): @freezegun.freeze_time("2022-01-01") def test_default_start_date(): - result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args(TEST_CONFIG_WITHOUT_START_DATE) + result = SourceZendeskSupport(config=TEST_CONFIG_WITHOUT_START_DATE, catalog=None, state=None).convert_config2stream_args( + TEST_CONFIG_WITHOUT_START_DATE + ) assert result["start_date"] == "2020-01-01T00:00:00Z" @@ -1047,14 +1049,15 @@ def test_read_non_json_error(requests_mock, caplog): read_full_refresh(stream) assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") + class TestTicketMetrics: @pytest.mark.parametrize( - "state, expected_implemented_stream", - [ - ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - ({}, StatelessTicketMetrics), - ] + "state, expected_implemented_stream", + [ + ({"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + ({}, StatelessTicketMetrics), + ], ) def test_get_implemented_stream(self, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1062,12 +1065,12 @@ def test_get_implemented_stream(self, state, expected_implemented_stream): assert isinstance(implemented_stream, expected_implemented_stream) @pytest.mark.parametrize( - "sync_mode, state, expected_implemented_stream", - [ - (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), - (SyncMode.full_refresh, {}, StatelessTicketMetrics), - (SyncMode.incremental, {}, StatelessTicketMetrics), - ] + "sync_mode, state, expected_implemented_stream", + [ + (SyncMode.incremental, {"_ab_updated_at": 1727334000}, StatefulTicketMetrics), + (SyncMode.full_refresh, {}, StatelessTicketMetrics), + (SyncMode.incremental, {}, StatelessTicketMetrics), + ], ) def test_stream_slices(self, sync_mode, state, expected_implemented_stream): stream = get_stream_instance(TicketMetrics, STREAM_ARGS) @@ -1081,7 +1084,12 @@ class TestStatefulTicketMetrics: [ ( {}, - {"tickets": [{"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}]}, + { + "tickets": [ + {"id": "13", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + {"id": "80", "generated_timestamp": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, + ] + }, [ {"ticket_id": "13", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, {"ticket_id": "80", "_ab_updated_at": pendulum.parse(STREAM_ARGS["start_date"]).int_timestamp}, @@ -1108,8 +1116,7 @@ def test_stream_slices(self, requests_mock, stream_state, response, expected_sli def test_read_with_error(self, requests_mock): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) requests_mock.get( - f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", - json={"error": "RecordNotFound", "description": "Not found"} + f"https://sandbox.zendesk.com/api/v2/tickets/13/metrics", json={"error": "RecordNotFound", "description": "Not found"} ) records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"ticket_id": "13"})) @@ -1119,12 +1126,12 @@ def test_read_with_error(self, requests_mock): @pytest.mark.parametrize( "status_code, response_action", ( - (200, ResponseAction.SUCCESS), - (404, ResponseAction.IGNORE), - (403, ResponseAction.IGNORE), - (500, ResponseAction.RETRY), - (429, ResponseAction.RATE_LIMITED), - ) + (200, ResponseAction.SUCCESS), + (404, ResponseAction.IGNORE), + (403, ResponseAction.IGNORE), + (500, ResponseAction.RETRY), + (429, ResponseAction.RATE_LIMITED), + ), ) def test_should_retry(self, status_code: int, response_action: bool): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) @@ -1135,37 +1142,62 @@ def test_should_retry(self, status_code: int, response_action: bool): @pytest.mark.parametrize( "current_stream_state, record_cursor_value, expected", [ - ({ "_ab_updated_at": 1727334000}, 1727420400, { "_ab_updated_at": 1727420400}), - ({ "_ab_updated_at": 1727334000}, 1700000000, { "_ab_updated_at": 1727334000}), - ] + ({"_ab_updated_at": 1727334000}, 1727420400, {"_ab_updated_at": 1727420400}), + ({"_ab_updated_at": 1727334000}, 1700000000, {"_ab_updated_at": 1727334000}), + ], ) def test_get_updated_state(self, current_stream_state, record_cursor_value, expected): stream = get_stream_instance(StatefulTicketMetrics, STREAM_ARGS) - latest_record = { "id": 1, "_ab_updated_at": record_cursor_value} + latest_record = {"id": 1, "_ab_updated_at": record_cursor_value} output_state = stream._get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) assert output_state == expected class TestStatelessTicketMetrics: @pytest.mark.parametrize( - "start_date, response, expected", - [ - ( - "2023-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp}, - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + "start_date, response, expected", + [ + ( + "2023-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ), - ( - "2024-01-01T00:00:00Z", - { "ticket_metrics": [{"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}]}, - [ - {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + }, + [ + { + "id": 1, + "ticket_id": 999, + "updated_at": "2023-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2023-02-01T00:00:00Z").int_timestamp, + }, + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + }, + ], + ), + ( + "2024-01-01T00:00:00Z", + { + "ticket_metrics": [ + {"id": 1, "ticket_id": 999, "updated_at": "2023-02-01T00:00:00Z"}, + {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z"}, ] - ) - ] + }, + [ + { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } + ], + ), + ], ) def test_parse_response(self, requests_mock, start_date, response, expected): stream_args = copy.deepcopy(STREAM_ARGS) @@ -1176,25 +1208,24 @@ def test_parse_response(self, requests_mock, start_date, response, expected): output = list(stream.parse_response(test_response, {})) assert expected == output - @pytest.mark.parametrize( - "has_more, expected", - [ - (True, {"page[after]": "nextpagecursor"}), - (False, None) - ] - ) + @pytest.mark.parametrize("has_more, expected", [(True, {"page[after]": "nextpagecursor"}), (False, None)]) def test_next_page_token(self, mocker, has_more, expected): stream = StatelessTicketMetrics(**STREAM_ARGS) ticket_metrics_response = mocker.Mock() - ticket_metrics_response.json.return_value = {"meta": { "after_cursor": "nextpagecursor", "has_more": has_more}} + ticket_metrics_response.json.return_value = {"meta": {"after_cursor": "nextpagecursor", "has_more": has_more}} result = stream.next_page_token(response=ticket_metrics_response) assert expected == result def test_get_updated_state(self): stream = StatelessTicketMetrics(**STREAM_ARGS) - stream._most_recently_updated_record = {"id": 2, "ticket_id": 1000, "updated_at": "2024-02-01T00:00:00Z", "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + stream._most_recently_updated_record = { + "id": 2, + "ticket_id": 1000, + "updated_at": "2024-02-01T00:00:00Z", + "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp, + } output_state = stream._get_updated_state(current_stream_state={}, latest_record={}) - expected_state = { "_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} + expected_state = {"_ab_updated_at": pendulum.parse("2024-02-01T00:00:00Z").int_timestamp} assert output_state == expected_state @@ -1236,13 +1267,7 @@ def test_validate_response_ticket_audits_handle_empty_response(audits_response, assert stream._validate_response(response_mock, {}) == expected -@pytest.mark.parametrize( - "initial_state_cursor_field", - [ - "generated_timestamp", - "_ab_updated_at" - ] -) +@pytest.mark.parametrize("initial_state_cursor_field", ["generated_timestamp", "_ab_updated_at"]) def test_ticket_metrics_state_migrataion(initial_state_cursor_field): state_migrator = TicketMetricsStateMigration() initial_state = {initial_state_cursor_field: 1672531200}