✨ Source JIRA: Add lookback window (#33532)

airbytehq · Dec 18, 2023 · 62ee5c5 · 62ee5c5
1 parent 677d5c2
commit 62ee5c5
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 6 deletions.
diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml
@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993
-  dockerImageTag: 0.13.0
+  dockerImageTag: 0.14.0
   dockerRepository: airbyte/source-jira
   documentationUrl: https://docs.airbyte.com/integrations/sources/jira
   githubIssueLabel: source-jira

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py
@@ -83,7 +83,7 @@ def _validate_and_transform(self, config: Mapping[str, Any]):
         start_date = config.get("start_date")
         if start_date:
             config["start_date"] = pendulum.parse(start_date)
-
+        config["lookback_window_minutes"] = pendulum.duration(minutes=config.get("lookback_window_minutes", 0))
         config["projects"] = config.get("projects", [])
         return config
 
@@ -145,7 +145,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         config = self._validate_and_transform(config)
         authenticator = self.get_authenticator(config)
         args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]}
-        incremental_args = {**args, "start_date": config.get("start_date")}
+        incremental_args = {
+            **args,
+            "start_date": config.get("start_date"),
+            "lookback_window_minutes": config.get("lookback_window_minutes"),
+        }
         issues_stream = Issues(**incremental_args)
         issue_fields_stream = IssueFields(**args)
         experimental_streams = []

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json
@@ -82,12 +82,22 @@
         "description": "Select fields to Expand the `Issues` stream when replicating with: ",
         "default": []
       },
+      "lookback_window_minutes": {
+        "title": "Lookback window",
+        "description": "When set to N, the connector will always refresh resources created within the past N minutes. By default, updated objects that are not newly created are not incrementally synced.",
+        "examples": [60],
+        "default": 0,
+        "minimum": 0,
+        "maximum": 576000,
+        "type": "integer",
+        "order": 5
+      },
       "enable_experimental_streams": {
         "type": "boolean",
         "title": "Enable Experimental Streams",
         "description": "Allow the use of experimental streams which rely on undocumented Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables for more info.",
         "default": false,
-        "order": 5
+        "order": 6
       }
     }
   }

diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py
@@ -135,8 +135,14 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]:
 
 
 class StartDateJiraStream(JiraStream, ABC):
-    def __init__(self, start_date: Optional[pendulum.DateTime] = None, **kwargs):
+    def __init__(
+        self,
+        start_date: Optional[pendulum.DateTime] = None,
+        lookback_window_minutes: pendulum.Duration = pendulum.duration(minutes=0),
+        **kwargs,
+    ):
         super().__init__(**kwargs)
+        self._lookback_window_minutes = lookback_window_minutes
         self._start_date = start_date
 
 
@@ -168,7 +174,7 @@ def _get_starting_point(self, stream_state: Mapping[str, Any]) -> Optional[pendu
         if stream_state:
             stream_state_value = stream_state.get(self.cursor_field)
             if stream_state_value:
-                stream_state_value = pendulum.parse(stream_state_value)
+                stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes
                 return safe_max(stream_state_value, self._start_date)
         return self._start_date
 

diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py
@@ -4,6 +4,7 @@
 
 import logging
 
+import pendulum
 import pytest
 import requests
 import responses
@@ -695,6 +696,22 @@ def test_issues_stream(config, mock_projects_responses_additional_project, mock_
     error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"]. Skipping for now. The user doesn't have permission to the project. Please grant the user to the project."
     assert error_message in caplog.messages
 
+@pytest.mark.parametrize(
+    "start_date, lookback_window, stream_state, expected_query",
+    [
+        (pendulum.parse("2023-09-09T00:00:00Z"), 0, None, None),
+        (None, 10, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:37'"),
+        (None, 0, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:47'")
+    ]
+)
+def test_issues_stream_jql_compare_date(config, start_date, lookback_window, stream_state, expected_query, caplog):
+    authenticator = SourceJira().get_authenticator(config=config)
+    args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"],
+            "lookback_window_minutes": pendulum.duration(minutes=lookback_window)}
+    stream = Issues(**args)
+    assert stream.jql_compare_date(stream_state) == expected_query
+
+
 
 @responses.activate
 def test_issue_comments_stream(config, mock_projects_responses, mock_issues_responses, issue_comments_response):

diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md
@@ -124,6 +124,7 @@ The Jira connector should not run into Jira API limitations under normal usage.
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                 |
 |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| 0.14.0  | 2023-12-15 | [33532](https://github.com/airbytehq/airbyte/pull/33532)   | Add lookback window                                                                                                     |
 | 0.13.0  | 2023-12-12 | [33353](https://github.com/airbytehq/airbyte/pull/33353)   | Fix check command to check access for all available streams                                                             |
 | 0.12.0  | 2023-12-01 | [33011](https://github.com/airbytehq/airbyte/pull/33011)   | Fix BoardIssues stream; increase number of retries for backoff policy to 10                                             |
 | 0.11.0  | 2023-11-29 | [32927](https://github.com/airbytehq/airbyte/pull/32927)   | Fix incremental syncs for stream Issues                                                                                 |