From dbb92fd37ed23f777422acdc29c57a02c8f7c98c Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 17:31:22 +0000 Subject: [PATCH 1/5] added unittest, got test to fail in the right way --- setup.py | 3 ++- tap_salesforce/salesforce/__init__.py | 3 +++ tests/unittests/test_auth_failure.py | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/test_auth_failure.py diff --git a/setup.py b/setup.py index 9199a744..e1214b8c 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ install_requires=[ 'requests==2.20.0', 'singer-python==5.10.0', - 'xmltodict==0.11.0' + 'xmltodict==0.11.0', + 'nose' ], entry_points=''' [console_scripts] diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index 4da64320..0fbe0e58 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -293,6 +293,9 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, return resp + def attempt_login(self): + print("hey there") + def login(self): if self.is_sandbox: login_url = 'https://test.salesforce.com/services/oauth2/token' diff --git a/tests/unittests/test_auth_failure.py b/tests/unittests/test_auth_failure.py new file mode 100644 index 00000000..bcb753d3 --- /dev/null +++ b/tests/unittests/test_auth_failure.py @@ -0,0 +1,23 @@ +import unittest +from unittest.mock import Mock +from tap_salesforce import Salesforce + +class TestAuthorizationHandling(unittest.TestCase): + def test_attempt_login_failure_with_retry(self): + """ + When we see an exception on the login, we expect the error to be raised after 3 tries + """ + mocked_service_caller = Mock() + mocked_service_caller._make_request = Mock(side_effect=Exception("this is an example auth exception")) + salesforce_object = Salesforce(default_start_date="2021-07-15T13:25:54Z") + with self.assertRaisesRegexp(Exception, "this is an example auth exception") as e: + salesforce_object.attempt_login() + self.assertEqual(3, mocked_service_caller._make_request.call_count) + + +""" +Traceback (most recent call last): + File "/opt/code/tap-salesforce/tests/unittests/test_auth_failure.py", line 13, in test_attempt_login_failure_with_retry + Salesforce.attempt_login() +AssertionError: "this is an example auth exception" does not match "attempt_login() missing 1 required positional argument: 'self'" +""" From f20e535269032aa41c5ed4e9ad26b771215ca0c9 Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 18:46:10 +0000 Subject: [PATCH 2/5] got rough draft of backoff working --- tap_salesforce/salesforce/__init__.py | 65 +++++++++++++++++---------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index 0fbe0e58..ee121191 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -133,6 +133,18 @@ def log_backoff_attempt(details): LOGGER.info("ConnectionError detected, triggering backoff: %d try", details.get("tries")) +def log_login_backoff(details):#what is passed in for details??? + # log issue + # reset login timer + # error_message = str(e) + # if resp is None and hasattr(e, 'response') and e.response is not None: #pylint:disable=no-member + # resp = e.response #pylint:disable=no-member + # # NB: requests.models.Response is always falsy here. It is false if status code >= 400 + # if isinstance(resp, requests.models.Response): + # error_message = error_message + ", Response from Salesforce: {}".format(resp.text) + # LOGGER.info(error_message) + LOGGER.info("Error logging in to Salesforce, triggering backoff: %d try", details.get("tries")) + def field_to_property_schema(field, mdata): # pylint:disable=too-many-branches property_schema = {} @@ -228,6 +240,7 @@ def __init__(self, self.login_timer = None self.data_url = "{}/services/data/v41.0/{}" self.pk_chunking = False + self.refresh_time = None # validate start_date singer_utils.strptime(default_start_date) @@ -273,6 +286,7 @@ def check_rest_quota_usage(self, headers): factor=2, on_backoff=log_backoff_attempt) def _make_request(self, http_method, url, headers=None, body=None, stream=False, params=None): + self.ensure_fresh_credentials() if http_method == "GET": LOGGER.info("Making %s request to %s with params: %s", http_method, url, params) resp = self.session.get(url, headers=headers, stream=stream, params=params) @@ -293,9 +307,20 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, return resp - def attempt_login(self): - print("hey there") + + + + def ensure_fresh_credentials(self): + if self.refresh_time == None or time.now() > self.refresh_time: + self.login() + return + # pylint: disable=too-many-arguments + @backoff.on_exception(backoff.constant, + Exception, + max_tries=3, + interval=REFRESH_TOKEN_EXPIRATION_PERIOD, + on_backoff=log_login_backoff) def login(self): if self.is_sandbox: login_url = 'https://test.salesforce.com/services/oauth2/token' @@ -308,27 +333,21 @@ def login(self): LOGGER.info("Attempting login via OAuth2") resp = None - try: - resp = self._make_request("POST", login_url, body=login_body, headers={"Content-Type": "application/x-www-form-urlencoded"}) - - LOGGER.info("OAuth2 login successful") - - auth = resp.json() - - self.access_token = auth['access_token'] - self.instance_url = auth['instance_url'] - except Exception as e: - error_message = str(e) - if resp is None and hasattr(e, 'response') and e.response is not None: #pylint:disable=no-member - resp = e.response #pylint:disable=no-member - # NB: requests.models.Response is always falsy here. It is false if status code >= 400 - if isinstance(resp, requests.models.Response): - error_message = error_message + ", Response from Salesforce: {}".format(resp.text) - raise Exception(error_message) from e - finally: - LOGGER.info("Starting new login timer") - self.login_timer = threading.Timer(REFRESH_TOKEN_EXPIRATION_PERIOD, self.login) - self.login_timer.start() + + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + time_before_login = time.time() + resp = self.session.post(login_url, headers=headers, data=body) + + LOGGER.info("OAuth2 login successful") + + auth = resp.json() + + self.access_token = auth['access_token'] + self.instance_url = auth['instance_url'] + self.refresh_time = time_before_login+REFRESH_TOKEN_EXPIRATION_PERIOD + + def describe(self, sobject=None): """Describes all objects or a specific object""" From 73b68136c911dd20366a44e285dee214ebe6bda9 Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 19:38:50 +0000 Subject: [PATCH 3/5] got unit test working --- tap_salesforce/salesforce/__init__.py | 8 ++------ tests/unittests/test_auth_failure.py | 15 +++------------ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index ee121191..06c08280 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -307,9 +307,6 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, return resp - - - def ensure_fresh_credentials(self): if self.refresh_time == None or time.now() > self.refresh_time: self.login() @@ -332,12 +329,11 @@ def login(self): LOGGER.info("Attempting login via OAuth2") - resp = None - headers = {"Content-Type": "application/x-www-form-urlencoded"} time_before_login = time.time() - resp = self.session.post(login_url, headers=headers, data=body) + + resp = self.session.post(login_url, headers=headers, data=login_body) LOGGER.info("OAuth2 login successful") diff --git a/tests/unittests/test_auth_failure.py b/tests/unittests/test_auth_failure.py index bcb753d3..527a7b5c 100644 --- a/tests/unittests/test_auth_failure.py +++ b/tests/unittests/test_auth_failure.py @@ -7,17 +7,8 @@ def test_attempt_login_failure_with_retry(self): """ When we see an exception on the login, we expect the error to be raised after 3 tries """ - mocked_service_caller = Mock() - mocked_service_caller._make_request = Mock(side_effect=Exception("this is an example auth exception")) salesforce_object = Salesforce(default_start_date="2021-07-15T13:25:54Z") + salesforce_object.session.post = Mock(side_effect=Exception("this is an example auth exception")) with self.assertRaisesRegexp(Exception, "this is an example auth exception") as e: - salesforce_object.attempt_login() - self.assertEqual(3, mocked_service_caller._make_request.call_count) - - -""" -Traceback (most recent call last): - File "/opt/code/tap-salesforce/tests/unittests/test_auth_failure.py", line 13, in test_attempt_login_failure_with_retry - Salesforce.attempt_login() -AssertionError: "this is an example auth exception" does not match "attempt_login() missing 1 required positional argument: 'self'" -""" + salesforce_object.login() + self.assertEqual(3, salesforce_object.session.post.call_count) From 6874603853332dd2f0095f1c45c7b9c855055b08 Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 19:42:52 +0000 Subject: [PATCH 4/5] cleaned up pylint --- tap_salesforce/salesforce/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index 06c08280..be6cafd3 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -308,9 +308,8 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, return resp def ensure_fresh_credentials(self): - if self.refresh_time == None or time.now() > self.refresh_time: + if self.refresh_time is None or time.time() > self.refresh_time: self.login() - return # pylint: disable=too-many-arguments @backoff.on_exception(backoff.constant, From 7b066f0746cefe85e3691849ed1c04bb7f4e05cd Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 19:45:30 +0000 Subject: [PATCH 5/5] removed trailing whitespace --- tap_salesforce/salesforce/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index be6cafd3..1e5b0547 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -342,8 +342,6 @@ def login(self): self.instance_url = auth['instance_url'] self.refresh_time = time_before_login+REFRESH_TOKEN_EXPIRATION_PERIOD - - def describe(self, sobject=None): """Describes all objects or a specific object""" headers = self._get_standard_headers()