From 23bd1f4534ab4765a09853156ea2bc36dacf0311 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Fri, 4 Aug 2023 00:09:34 +0000 Subject: [PATCH 01/10] Testcase to test the api quota limit --- tests/test_salesforce_api_per_run_quota.py | 53 ++++++++++++++++++++++ tests/test_salesforce_api_total_quota.py | 50 ++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 tests/test_salesforce_api_per_run_quota.py create mode 100644 tests/test_salesforce_api_total_quota.py diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py new file mode 100644 index 00000000..18aaa19d --- /dev/null +++ b/tests/test_salesforce_api_per_run_quota.py @@ -0,0 +1,53 @@ +""" +Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated +""" +import copy +from tap_tester.base_suite_tests.api_quota_test import APIQuotaTest +from sfbase import SFBaseTest +from tap_tester import runner, menagerie + + +class SFAPIQuota(APIQuotaTest, SFBaseTest): + """Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated + TODO + This test coveres only BULK api and just couple of streams. We have separate cards to cover these cases + TDL-23653: [tap-salesforce]: QA - Add all the streams for the all_fields test + TDL-23654: [tap-salesforce]: QA - Add all-fields testcase for REST API streams + """ + + salesforce_api = 'BULK' + expected_error = "Terminating replication due to allotted quota of 1.0% per replication" + + @staticmethod + def name(): + return "tt_sf_api_quota" + + def get_properties(self, original: bool = True): + """Configuration properties required for the tap.""" + return_value = { + 'start_date': '2000-11-23T00:00:00Z', + 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', + 'select_fields_by_default': 'true', + 'quota_percent_total': '85', + 'quota_percent_per_run' : '1', + 'api_type': self.salesforce_api, + 'is_sandbox': 'false' + } + if original: + return return_value + + # This test needs the new connections start date to be larger than the default + assert self.start_date > return_value["start_date"] + + return_value["start_date"] = self.start_date + return return_value + + def streams_to_test(self): + return self.expected_streams().difference( self.streams_to_exclude ) + + def test_all_streams_sync_records(self): + for stream in self.streams_to_test(): + with self.subTest(stream=stream): + + # gather results + record_count = self.record_count_by_stream diff --git a/tests/test_salesforce_api_total_quota.py b/tests/test_salesforce_api_total_quota.py new file mode 100644 index 00000000..027c0e5d --- /dev/null +++ b/tests/test_salesforce_api_total_quota.py @@ -0,0 +1,50 @@ +""" +Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated +""" +import copy +from tap_tester.base_suite_tests.api_quota_test import APIQuotaTest +from sfbase import SFBaseTest +from tap_tester import runner, menagerie + + +class SFAPIQuota(APIQuotaTest, SFBaseTest): + """Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated + TODO + This test coveres only BULK api and just couple of streams. We have separate cards to cover these cases + TDL-23653: [tap-salesforce]: QA - Add all the streams for the all_fields test + TDL-23654: [tap-salesforce]: QA - Add all-fields testcase for REST API streams + """ + + salesforce_api = 'BULK' + expected_error = "Terminating replication to not continue past configured percentage of 2.0% total quota" + + @staticmethod + def name(): + return "tt_sf_api_quota_total" + + def get_properties(self, original: bool = True): + """Configuration properties required for the tap.""" + return_value = { + 'start_date': '2000-11-23T00:00:00Z', + 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', + 'select_fields_by_default': 'true', + 'quota_percent_total': '2', + 'quota_percent_per_run' : '1', + 'api_type': self.salesforce_api, + 'is_sandbox': 'false' + } + if original: + return return_value + + # This test needs the new connections start date to be larger than the default + assert self.start_date > return_value["start_date"] + + return_value["start_date"] = self.start_date + return return_value + + def test_all_streams_sync_records(self): + for stream in self.streams_to_test(): + with self.subTest(stream=stream): + + # gather results + record_count = self.record_count_by_stream From 826de781a89295834868c14985d6785cd1ba262f Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Tue, 15 Aug 2023 00:03:12 +0000 Subject: [PATCH 02/10] Set variables for properties to use in quota tests --- tests/sfbase.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/sfbase.py b/tests/sfbase.py index 46e0cb8b..b0d0dc97 100644 --- a/tests/sfbase.py +++ b/tests/sfbase.py @@ -14,7 +14,14 @@ class SFBaseTest(BaseCase): + """ + Set these variables for the properties to use in the other tests + """ + salesforce_api = "BULK" + total_quota = '80' + per_run_quota = None + start_date = '2020-11-23T00:00:00Z' @staticmethod def tap_name(): @@ -29,10 +36,11 @@ def get_type(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { - 'start_date': '2020-11-23T00:00:00Z', + 'start_date': self.start_date, 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', 'select_fields_by_default': 'true', - 'quota_percent_total': '80', + 'quota_percent_total': self.total_quota, + 'quota_percent_per_run' : self.per_run_quota, 'api_type': self.salesforce_api, 'is_sandbox': 'false' } @@ -828,6 +836,14 @@ def expected_metadata(): 'WorkStepTemplateShare': incremental_last_modified, } + def expected_streams(self): + """A set of expected stream names""" + streams = set(self.expected_metadata().keys()) + + if self.salesforce_api == 'BULK': + return streams.difference(self.rest_only_streams()) + return streams + def rest_only_streams(self): """A group of streams that is only discovered when the REST API is in use.""" return { From a24377d54f8560f4497f604cdccda3234d2dd0f6 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Tue, 15 Aug 2023 00:05:17 +0000 Subject: [PATCH 03/10] Removed get_properties and modified to have all the other functions within the tests --- tests/test_salesforce_api_per_run_quota.py | 92 +++++++++++++--------- tests/test_salesforce_api_total_quota.py | 79 ++++++++++--------- 2 files changed, 97 insertions(+), 74 deletions(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 18aaa19d..21383f1b 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -1,53 +1,69 @@ -""" -Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated -""" -import copy -from tap_tester.base_suite_tests.api_quota_test import APIQuotaTest from sfbase import SFBaseTest -from tap_tester import runner, menagerie +from tap_tester import connections, runner, menagerie +from tap_tester.logger import LOGGER -class SFAPIQuota(APIQuotaTest, SFBaseTest): - """Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated - TODO - This test coveres only BULK api and just couple of streams. We have separate cards to cover these cases - TDL-23653: [tap-salesforce]: QA - Add all the streams for the all_fields test - TDL-23654: [tap-salesforce]: QA - Add all-fields testcase for REST API streams +class SFAPIQuota(SFBaseTest): + """ + https://jira.talendforge.org/browse/TDL-23431 + This testcase makes sure we are able to configure the tap with specific + per_run quota for apis and verifies if tap is working as per the + api quota limit set. """ - salesforce_api = 'BULK' - expected_error = "Terminating replication due to allotted quota of 1.0% per replication" + """ + Set a start date well past so that we have enough data to sync to hit the limit + set a low quota to make the sync fail + some streams are excluded as they don't have any data + """ + + start_date = '2000-11-23T00:00:00Z' + per_run_quota = '1' + streams_to_exclude = { + 'DatacloudAddress', + 'DatacloudCompany', + 'DatacloudContact', + 'DatacloudDandBCompany', + 'DatacloudOwnedEntity', + 'DatacloudPurchaseUsage', + 'FieldSecurityClassification', + 'ServiceAppointmentStatus', + 'WorkOrderLineItemStatus', + 'WorkOrderStatus', + 'ShiftStatus', + 'WorkStepStatus', + } @staticmethod def name(): return "tt_sf_api_quota" - def get_properties(self, original: bool = True): - """Configuration properties required for the tap.""" - return_value = { - 'start_date': '2000-11-23T00:00:00Z', - 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', - 'select_fields_by_default': 'true', - 'quota_percent_total': '85', - 'quota_percent_per_run' : '1', - 'api_type': self.salesforce_api, - 'is_sandbox': 'false' - } - if original: - return return_value + def streams_to_test(self): + return self.expected_streams().difference( self.streams_to_exclude ) - # This test needs the new connections start date to be larger than the default - assert self.start_date > return_value["start_date"] + @staticmethod + def streams_to_selected_fields(): + # Select all fields in the selected catalogs (streams) + return {} - return_value["start_date"] = self.start_date - return return_value + def test_api_per_run_quota(self): + """ + Run the tap in check mode and verify it returns the error for quota limit reached. If not, proceed to sync mode. + """ + expected_per_run_error = "Terminating replication due to allotted quota" + conn_id = connections.ensure_connection(self) - def streams_to_test(self): - return self.expected_streams().difference( self.streams_to_exclude ) + found_catalogs = self.run_and_verify_check_mode( conn_id) + + # table and field selection + test_catalogs = [catalog for catalog in found_catalogs + if catalog.get('stream_name') in self.streams_to_test()] + + # non_selected_fields are none + self.perform_and_verify_table_and_field_selection(conn_id, test_catalogs) + + with self.assertRaises(Exception) as ex: + record_count_by_stream = self.run_and_verify_sync_mode(conn_id) - def test_all_streams_sync_records(self): - for stream in self.streams_to_test(): - with self.subTest(stream=stream): + self.assertIn(expected_per_run_error, str(ex.exception) ) - # gather results - record_count = self.record_count_by_stream diff --git a/tests/test_salesforce_api_total_quota.py b/tests/test_salesforce_api_total_quota.py index 027c0e5d..6d9fb0d2 100644 --- a/tests/test_salesforce_api_total_quota.py +++ b/tests/test_salesforce_api_total_quota.py @@ -1,50 +1,57 @@ -""" -Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated -""" -import copy -from tap_tester.base_suite_tests.api_quota_test import APIQuotaTest from sfbase import SFBaseTest -from tap_tester import runner, menagerie +from tap_tester import runner, menagerie, LOGGER, connections -class SFAPIQuota(APIQuotaTest, SFBaseTest): - """Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated - TODO - This test coveres only BULK api and just couple of streams. We have separate cards to cover these cases - TDL-23653: [tap-salesforce]: QA - Add all the streams for the all_fields test - TDL-23654: [tap-salesforce]: QA - Add all-fields testcase for REST API streams +class SFAPIQuota( SFBaseTest): + """ + https://jira.talendforge.org/browse/TDL-23431 + This testcase makes sure we are able to configure the tap with specific + total quota for apis and verifies if tap is working as per the + api quota limit set. """ - salesforce_api = 'BULK' - expected_error = "Terminating replication to not continue past configured percentage of 2.0% total quota" + """ + Set a start date well past so that we have enough data to sync to hit the limit + set a low quota to make the sync fail + some streams are excluded as they don't have any data + """ + + start_date = '2000-11-23T00:00:00Z' + total_quota = '1' + streams_to_exclude = { + 'DatacloudAddress', + 'DatacloudCompany', + 'DatacloudContact', + 'DatacloudDandBCompany', + 'DatacloudOwnedEntity', + 'DatacloudPurchaseUsage', + 'FieldSecurityClassification', + 'ServiceAppointmentStatus', + 'WorkOrderLineItemStatus', + 'WorkOrderStatus', + 'ShiftStatus', + 'WorkStepStatus', + } @staticmethod def name(): return "tt_sf_api_quota_total" - def get_properties(self, original: bool = True): - """Configuration properties required for the tap.""" - return_value = { - 'start_date': '2000-11-23T00:00:00Z', - 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', - 'select_fields_by_default': 'true', - 'quota_percent_total': '2', - 'quota_percent_per_run' : '1', - 'api_type': self.salesforce_api, - 'is_sandbox': 'false' - } - if original: - return return_value + def streams_to_test(self): + return self.expected_streams().difference( self.streams_to_exclude ) + + def test_api_total_quota(self): + """ + Run the tap in check mode and verify it returns the error for quota limit reached. + """ + expected_total_quota_error = "Terminating replication to not continue past configured percentage of 1.0% total quota" - # This test needs the new connections start date to be larger than the default - assert self.start_date > return_value["start_date"] + conn_id = connections.ensure_connection(self) - return_value["start_date"] = self.start_date - return return_value + # run check mode + # Run a check job using orchestrator (discovery) + with self.assertRaises(Exception) as ex: + check_job_name = self.run_and_verify_check_mode( conn_id) - def test_all_streams_sync_records(self): - for stream in self.streams_to_test(): - with self.subTest(stream=stream): + self.assertIn(expected_total_quota_error, str(ex.exception) ) - # gather results - record_count = self.record_count_by_stream From 7fbe04ec4e2ce377b42c50e2ed2ae509bfcf141e Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Wed, 30 Aug 2023 01:13:20 +0000 Subject: [PATCH 04/10] Bumped up the quota limit to 95 --- tests/base.py | 2 +- tests/sfbase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/base.py b/tests/base.py index 2938a85e..ed5fc5b6 100644 --- a/tests/base.py +++ b/tests/base.py @@ -54,7 +54,7 @@ def get_properties(self, original: bool = True): 'start_date': '2020-11-23T00:00:00Z', 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', 'select_fields_by_default': 'true', - 'quota_percent_total': '80', + 'quota_percent_total': '95', 'api_type': self.salesforce_api, 'is_sandbox': 'false' } diff --git a/tests/sfbase.py b/tests/sfbase.py index b0d0dc97..fdb4a1af 100644 --- a/tests/sfbase.py +++ b/tests/sfbase.py @@ -19,7 +19,7 @@ class SFBaseTest(BaseCase): """ salesforce_api = "BULK" - total_quota = '80' + total_quota = '95' per_run_quota = None start_date = '2020-11-23T00:00:00Z' From 312f1c8a86485ab0d718935a880864331fd2d3d1 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Wed, 30 Aug 2023 01:14:34 +0000 Subject: [PATCH 05/10] Handled check mode failure --- tests/test_salesforce_api_per_run_quota.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 21383f1b..5a7e1c35 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -51,9 +51,15 @@ def test_api_per_run_quota(self): Run the tap in check mode and verify it returns the error for quota limit reached. If not, proceed to sync mode. """ expected_per_run_error = "Terminating replication due to allotted quota" + expected_total_quota_error = "Terminating replication to not continue past configured percentage " conn_id = connections.ensure_connection(self) - found_catalogs = self.run_and_verify_check_mode( conn_id) + # run check mode + # Run a check job using orchestrator (discovery) + with self.assertRaises(Exception) as ex: + found_catalogs = self.run_and_verify_check_mode( conn_id) + + self.assertIn(expected_total_quota_error, str(ex.exception) ) # table and field selection test_catalogs = [catalog for catalog in found_catalogs From 8fb3e9ba696103456220df63ee55faf18fbc4301 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Thu, 31 Aug 2023 05:30:20 +0000 Subject: [PATCH 06/10] Removed white space and duplicate comment --- tests/test_salesforce_api_per_run_quota.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 5a7e1c35..d1dfed97 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -49,12 +49,12 @@ def streams_to_selected_fields(): def test_api_per_run_quota(self): """ Run the tap in check mode and verify it returns the error for quota limit reached. If not, proceed to sync mode. + For the sync mode, we have a higher total quota set, so it is unlikely to hit the total quota. Noticed that per run quota limit reaches only during the sync mode. """ expected_per_run_error = "Terminating replication due to allotted quota" - expected_total_quota_error = "Terminating replication to not continue past configured percentage " + expected_total_quota_error = "Terminating replication to not continue past configured percentage" conn_id = connections.ensure_connection(self) - # run check mode # Run a check job using orchestrator (discovery) with self.assertRaises(Exception) as ex: found_catalogs = self.run_and_verify_check_mode( conn_id) From 245e56aced4dfe5823da9bda3d16355f4be8c044 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Fri, 1 Sep 2023 01:44:06 +0000 Subject: [PATCH 07/10] Removed extra spaces --- tests/test_salesforce_api_per_run_quota.py | 8 ++++---- tests/test_salesforce_api_total_quota.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index d1dfed97..84cbcf27 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -39,7 +39,7 @@ def name(): return "tt_sf_api_quota" def streams_to_test(self): - return self.expected_streams().difference( self.streams_to_exclude ) + return self.expected_streams().difference(self.streams_to_exclude) @staticmethod def streams_to_selected_fields(): @@ -57,9 +57,9 @@ def test_api_per_run_quota(self): # Run a check job using orchestrator (discovery) with self.assertRaises(Exception) as ex: - found_catalogs = self.run_and_verify_check_mode( conn_id) + found_catalogs = self.run_and_verify_check_mode(conn_id) - self.assertIn(expected_total_quota_error, str(ex.exception) ) + self.assertIn(expected_total_quota_error, str(ex.exception)) # table and field selection test_catalogs = [catalog for catalog in found_catalogs @@ -71,5 +71,5 @@ def test_api_per_run_quota(self): with self.assertRaises(Exception) as ex: record_count_by_stream = self.run_and_verify_sync_mode(conn_id) - self.assertIn(expected_per_run_error, str(ex.exception) ) + self.assertIn(expected_per_run_error, str(ex.exception)) diff --git a/tests/test_salesforce_api_total_quota.py b/tests/test_salesforce_api_total_quota.py index 6d9fb0d2..b92217e9 100644 --- a/tests/test_salesforce_api_total_quota.py +++ b/tests/test_salesforce_api_total_quota.py @@ -2,7 +2,7 @@ from tap_tester import runner, menagerie, LOGGER, connections -class SFAPIQuota( SFBaseTest): +class SFAPIQuota(SFBaseTest): """ https://jira.talendforge.org/browse/TDL-23431 This testcase makes sure we are able to configure the tap with specific @@ -38,7 +38,7 @@ def name(): return "tt_sf_api_quota_total" def streams_to_test(self): - return self.expected_streams().difference( self.streams_to_exclude ) + return self.expected_streams().difference(self.streams_to_exclude) def test_api_total_quota(self): """ @@ -51,7 +51,7 @@ def test_api_total_quota(self): # run check mode # Run a check job using orchestrator (discovery) with self.assertRaises(Exception) as ex: - check_job_name = self.run_and_verify_check_mode( conn_id) + check_job_name = self.run_and_verify_check_mode(conn_id) - self.assertIn(expected_total_quota_error, str(ex.exception) ) + self.assertIn(expected_total_quota_error, str(ex.exception)) From b867a8fff4b307f97679a0e792fd7187e75b1f98 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Tue, 5 Sep 2023 13:59:35 +0000 Subject: [PATCH 08/10] Added try/catch to handle exception for check mode failure --- tests/test_salesforce_api_per_run_quota.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 84cbcf27..2452d8e7 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -56,10 +56,10 @@ def test_api_per_run_quota(self): conn_id = connections.ensure_connection(self) # Run a check job using orchestrator (discovery) - with self.assertRaises(Exception) as ex: + try: found_catalogs = self.run_and_verify_check_mode(conn_id) - - self.assertIn(expected_total_quota_error, str(ex.exception)) + except Exception as ex: + raise ex # table and field selection test_catalogs = [catalog for catalog in found_catalogs From 1483ce78ac3ea9e8500bafe5af2a61be5ef5c346 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Tue, 5 Sep 2023 16:15:47 +0000 Subject: [PATCH 09/10] Added assertion for exception in check mode --- tests/test_salesforce_api_per_run_quota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 2452d8e7..584301df 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -59,7 +59,7 @@ def test_api_per_run_quota(self): try: found_catalogs = self.run_and_verify_check_mode(conn_id) except Exception as ex: - raise ex + self.assertIn(expected_per_run_error, str(ex.exception)) # table and field selection test_catalogs = [catalog for catalog in found_catalogs From 930c66b4dc993c1c4a9d1dcfa8f3892058ddb296 Mon Sep 17 00:00:00 2001 From: Bhuvana Venkataraman Date: Tue, 5 Sep 2023 16:16:14 +0000 Subject: [PATCH 10/10] Removed duplicate comment --- tests/test_salesforce_api_total_quota.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_salesforce_api_total_quota.py b/tests/test_salesforce_api_total_quota.py index b92217e9..b6cd10bd 100644 --- a/tests/test_salesforce_api_total_quota.py +++ b/tests/test_salesforce_api_total_quota.py @@ -48,7 +48,6 @@ def test_api_total_quota(self): conn_id = connections.ensure_connection(self) - # run check mode # Run a check job using orchestrator (discovery) with self.assertRaises(Exception) as ex: check_job_name = self.run_and_verify_check_mode(conn_id)