From 4df80965f2cd9eba7ab4967a49665efecd08a8a6 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Sat, 11 Nov 2023 11:46:11 +0400 Subject: [PATCH] Source Zendesk Support: increase test coverage (#32440) Co-authored-by: roman-yermilov-gl --- .../integration_tests/expected_records.jsonl | 6 +- .../source-zendesk-support/metadata.yaml | 2 +- .../source_zendesk_support/streams.py | 67 ++----------------- .../unit_tests/test_backoff_on_rate_limit.py | 9 ++- .../unit_tests/unit_test.py | 44 +++++++++++- docs/integrations/sources/zendesk-support.md | 1 + 6 files changed, 57 insertions(+), 72 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index c195fc5166bb..615d50bb3ebc 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -41,9 +41,9 @@ {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860081} {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860083} {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1697714860085} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-10-17T14:27:52Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1697714861785} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1697714861787} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1697714861788} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-11-06T15:01:40Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1699646404810} {"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863384} {"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} {"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} diff --git a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml index a13d1d5e9029..425d8ee26e57 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 - dockerImageTag: 2.2.0 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-zendesk-support documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support githubIssueLabel: source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index b667830c98a6..bff5b5fdfed1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -351,13 +351,14 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + next_page_token = next_page_token or {} + parsed_state = self.check_stream_state(stream_state) + params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} # check "start_time" is not in the future params["start_time"] = self.check_start_time_param(params["start_time"]) if self.sideload_param: params["include"] = self.sideload_param if next_page_token: - params.pop("start_time", None) params.update(next_page_token) return params @@ -398,23 +399,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, response_json = response.json() return None if response_json.get(END_OF_STREAM_KEY, True) else {"start_time": response_json.get("end_time")} - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - @property def update_event_from_record(self) -> bool: """Returns True/False based on list_entities_from_event property""" @@ -455,46 +439,12 @@ class Users(SourceZendeskIncrementalExportStream): def path(self, **kwargs) -> str: return "incremental/users/cursor.json" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - class Organizations(SourceZendeskIncrementalExportStream): """Organizations stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/""" response_list_name: str = "organizations" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - class Posts(CursorPaginationZendeskSupportStream): """Posts stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts""" @@ -894,16 +844,7 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"sort_by": "updated_at", "sort_order": "asc", "start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params + return {"sort_by": "updated_at", "sort_order": "asc", **super().request_params(stream_state, stream_slice, next_page_token)} class ArticleVotes(AbstractVotes, HttpSubStream): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py index 89349107b60b..0c17e1bcc722 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py @@ -25,10 +25,13 @@ def prepare_config(config: Dict): return SourceZendeskSupport().convert_config2stream_args(config) -@pytest.mark.parametrize("retry_after, expected", [({}, None), ({"Retry-After": "5"}, 5), ({"Retry-After": "5, 4"}, 5)]) -def test_backoff(requests_mock, config, retry_after, expected): +@pytest.mark.parametrize( + "x_rate_limit, retry_after, expected", + [("60", {}, 1), ("0", {}, None), ("0", {"Retry-After": "5"}, 5), ("0", {"Retry-After": "5, 4"}, 5)], +) +def test_backoff(requests_mock, config, x_rate_limit, retry_after, expected): """ """ - test_response_header = {"X-Rate-Limit": "0"} | retry_after + test_response_header = {"X-Rate-Limit": x_rate_limit} | retry_after test_response_json = {"count": {"value": 1, "refreshed_at": "2022-03-29T10:10:51+00:00"}} # create client diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 223515426aad..cdb9a4d84d53 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -22,6 +22,10 @@ END_OF_STREAM_KEY, LAST_END_TIME_KEY, AccountAttributes, + ArticleComments, + ArticleCommentVotes, + Articles, + ArticleVotes, AttributeDefinitions, AuditLogs, BaseZendeskSupportStream, @@ -70,6 +74,13 @@ "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, } +# raw old config +TEST_OLD_CONFIG = { + "auth_method": {"auth_method": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, + "subdomain": "sandbox", + "start_date": "2021-06-01T00:00:00Z", +} + TEST_CONFIG_WITHOUT_START_DATE = { "subdomain": "sandbox", "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, @@ -131,8 +142,12 @@ def test_default_start_date(): @pytest.mark.parametrize( "config, expected", - [(TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), (TEST_CONFIG_OAUTH, "test_access_token")], - ids=["api_token", "oauth"], + [ + (TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + (TEST_CONFIG_OAUTH, "test_access_token"), + (TEST_OLD_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + ], + ids=["api_token", "oauth", "old_config"], ) def test_get_authenticator(config, expected): # we expect base64 from creds input @@ -320,6 +335,12 @@ def test_streams(self, expected_stream_cls): if expected_stream_cls in streams: assert isinstance(stream, expected_stream_cls) + def test_ticket_forms_exception_stream(self): + with patch.object(TicketForms, "read_records", return_value=[{}]) as mocked_records: + mocked_records.side_effect = Exception("The error") + streams = SourceZendeskSupport().streams(TEST_CONFIG) + assert not any([isinstance(stream, TicketForms) for stream in streams]) + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -756,10 +777,12 @@ def test_next_page_token(self, requests_mock, stream_cls): [ (Users, {"start_time": 1622505600}), (Tickets, {"start_time": 1622505600}), + (Articles, {"sort_by": "updated_at", "sort_order": "asc", "start_time": 1622505600}), ], ids=[ "Users", "Tickets", + "Articles", ], ) def test_request_params(self, stream_cls, expected): @@ -787,6 +810,23 @@ def test_parse_response(self, requests_mock, stream_cls): output = list(stream.parse_response(test_response)) assert expected == output + @pytest.mark.parametrize( + "stream_cls, stream_slice, expected_path", + [ + (ArticleVotes, {"parent": {"id": 1}}, "help_center/articles/1/votes"), + (ArticleComments, {"parent": {"id": 1}}, "help_center/articles/1/comments"), + (ArticleCommentVotes, {"parent": {"id": 1, "source_id": 1}}, "help_center/articles/1/comments/1/votes"), + ], + ids=[ + "ArticleVotes_path", + "ArticleComments_path", + "ArticleCommentVotes_path", + ], + ) + def test_path(self, stream_cls, stream_slice, expected_path): + stream = stream_cls(**STREAM_ARGS) + assert stream.path(stream_slice=stream_slice) == expected_path + class TestSourceZendeskSupportTicketEventsExportStream: @pytest.mark.parametrize( diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 053f9c1a3252..5f194f8322e5 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -134,6 +134,7 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.2.1` | 2023-11-10 | [32440](https://github.com/airbytehq/airbyte/pull/32440) | Made refactoring to improve code maintainability | | `2.2.0` | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extended the `CustomRoles` stream schema | | `2.1.1` | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | | `2.1.0` | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Added new field `reply_time_in_seconds` to the `Ticket Metrics` stream schema |