From 62ee5c52844ebd569607ea6944c66d007ad42758 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:44:08 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Source=20JIRA:=20Add=20lookback=20w?= =?UTF-8?q?indow=20(#33532)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-jira/metadata.yaml | 2 +- .../source-jira/source_jira/source.py | 8 ++++++-- .../source-jira/source_jira/spec.json | 12 +++++++++++- .../source-jira/source_jira/streams.py | 10 ++++++++-- .../source-jira/unit_tests/test_streams.py | 17 +++++++++++++++++ docs/integrations/sources/jira.md | 1 + 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index 7fd64d0667f3..255249d84092 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 0.13.0 + dockerImageTag: 0.14.0 dockerRepository: airbyte/source-jira documentationUrl: https://docs.airbyte.com/integrations/sources/jira githubIssueLabel: source-jira diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index 85ebd5e571ee..a34a74949070 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -83,7 +83,7 @@ def _validate_and_transform(self, config: Mapping[str, Any]): start_date = config.get("start_date") if start_date: config["start_date"] = pendulum.parse(start_date) - + config["lookback_window_minutes"] = pendulum.duration(minutes=config.get("lookback_window_minutes", 0)) config["projects"] = config.get("projects", []) return config @@ -145,7 +145,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} - incremental_args = {**args, "start_date": config.get("start_date")} + incremental_args = { + **args, + "start_date": config.get("start_date"), + "lookback_window_minutes": config.get("lookback_window_minutes"), + } issues_stream = Issues(**incremental_args) issue_fields_stream = IssueFields(**args) experimental_streams = [] diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json index 93d9fa13c71a..fff67ec349e6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/spec.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/spec.json @@ -82,12 +82,22 @@ "description": "Select fields to Expand the `Issues` stream when replicating with: ", "default": [] }, + "lookback_window_minutes": { + "title": "Lookback window", + "description": "When set to N, the connector will always refresh resources created within the past N minutes. By default, updated objects that are not newly created are not incrementally synced.", + "examples": [60], + "default": 0, + "minimum": 0, + "maximum": 576000, + "type": "integer", + "order": 5 + }, "enable_experimental_streams": { "type": "boolean", "title": "Enable Experimental Streams", "description": "Allow the use of experimental streams which rely on undocumented Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables for more info.", "default": false, - "order": 5 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 6b6cc0af9759..c11bc28f28d5 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -135,8 +135,14 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class StartDateJiraStream(JiraStream, ABC): - def __init__(self, start_date: Optional[pendulum.DateTime] = None, **kwargs): + def __init__( + self, + start_date: Optional[pendulum.DateTime] = None, + lookback_window_minutes: pendulum.Duration = pendulum.duration(minutes=0), + **kwargs, + ): super().__init__(**kwargs) + self._lookback_window_minutes = lookback_window_minutes self._start_date = start_date @@ -168,7 +174,7 @@ def _get_starting_point(self, stream_state: Mapping[str, Any]) -> Optional[pendu if stream_state: stream_state_value = stream_state.get(self.cursor_field) if stream_state_value: - stream_state_value = pendulum.parse(stream_state_value) + stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes return safe_max(stream_state_value, self._start_date) return self._start_date diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index 78a0fc31c946..7b5f693a4fee 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -4,6 +4,7 @@ import logging +import pendulum import pytest import requests import responses @@ -695,6 +696,22 @@ def test_issues_stream(config, mock_projects_responses_additional_project, mock_ error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"]. Skipping for now. The user doesn't have permission to the project. Please grant the user to the project." assert error_message in caplog.messages +@pytest.mark.parametrize( + "start_date, lookback_window, stream_state, expected_query", + [ + (pendulum.parse("2023-09-09T00:00:00Z"), 0, None, None), + (None, 10, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:37'"), + (None, 0, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:47'") + ] +) +def test_issues_stream_jql_compare_date(config, start_date, lookback_window, stream_state, expected_query, caplog): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"], + "lookback_window_minutes": pendulum.duration(minutes=lookback_window)} + stream = Issues(**args) + assert stream.jql_compare_date(stream_state) == expected_query + + @responses.activate def test_issue_comments_stream(config, mock_projects_responses, mock_issues_responses, issue_comments_response): diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index 3477af0fcc03..2c25b8b88d6d 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -124,6 +124,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 0.14.0 | 2023-12-15 | [33532](https://github.com/airbytehq/airbyte/pull/33532) | Add lookback window | | 0.13.0 | 2023-12-12 | [33353](https://github.com/airbytehq/airbyte/pull/33353) | Fix check command to check access for all available streams | | 0.12.0 | 2023-12-01 | [33011](https://github.com/airbytehq/airbyte/pull/33011) | Fix BoardIssues stream; increase number of retries for backoff policy to 10 | | 0.11.0 | 2023-11-29 | [32927](https://github.com/airbytehq/airbyte/pull/32927) | Fix incremental syncs for stream Issues |