Merge pull request #7 from harrystech/ynaim94/add-end-date

Add end date

README.md

@@ -1,6 +1,6 @@
 # tap-gladly
 
-`tap-gladly` is a Singer tap for gladly.
+`tap-gladly` is a Singer tap for Gladly and the Export API
 
 Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps.
 
@@ -40,7 +40,8 @@ pipx install git+https://github.com/ORG_NAME/tap-gladly.git@main
 | username            | True     | None    | The username to authenticate against the API service |
 | password            | True     | None    | The username to authenticate against the API service |
 | project_ids         | False    | None    | Project IDs to replicate |
-| start_date          | True     | None    | The earliest record date to sync, format %Y-%m-%dT%H:%M:%SZ |
+| start_date          | True     | None    | The earliest job date to sync, format %Y-%m-%dT%H:%M:%SZ |
+| end_date            | False    | None    | The latest job date to sync, format %Y-%m-%dT%H:%M:%SZ |
 | api_url_base        | True     | None    | The url for the API service |
 | stream_maps         | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
 | stream_map_config   | False    | None    | User-defined config values to be used within map expressions. |

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-gladly"
-version = "0.2.0"
+version = "0.3.0"
 description = "`tap-gladly` is a Singer tap for gladly, built with the Meltano SDK for Singer Taps."
 authors = ["harrystech"]
 keywords = [

tap_gladly/streams.py

@@ -26,10 +26,13 @@ class ExportCompletedJobsStream(gladlyStream):
 
     def post_process(self, row, context):
         """Filter jobs that finished before start_date."""
-        if pendulum.parse(row["parameters"]["endAt"]) >= pendulum.parse(
-            self.config["start_date"]
-        ):
-            return row
+        job_completion_date = pendulum.parse(row["parameters"]["endAt"])
+        if pendulum.parse(self.config["start_date"]) <= job_completion_date:
+            if "end_date" in self.config:
+                if pendulum.parse(self.config["end_date"]) >= job_completion_date:
+                    return row
+            else:
+                return row
         return
 
     def get_child_context(self, record: dict, context: Optional[dict]) -> dict:

tap_gladly/tap.py

@@ -69,7 +69,12 @@ class Tapgladly(Tap):
             "start_date",
             th.DateTimeType,
             required=True,
-            description="The earliest record date to sync, format %Y-%m-%dT%H:%M:%SZ",
+            description="The earliest job date to sync, format %Y-%m-%dT%H:%M:%SZ",
+        ),
+        th.Property(
+            "end_date",
+            th.DateTimeType,
+            description="The latest job date to sync, format %Y-%m-%dT%H:%M:%SZ",
         ),
         th.Property(
             "api_url_base",

tap_gladly/tests/test_streams.py

@@ -41,6 +41,37 @@ def test_started_at():
     assert export_jobs_stream.post_process(after_row, None)
 
 
+def test_data_interval():
+    tap_gladly = Tapgladly(
+        config=dict(
+            SAMPLE_CONFIG,
+            start_date=(pendulum.now() - datetime.timedelta(days=2)).isoformat(),
+            end_date=(pendulum.now() - datetime.timedelta(days=1)).isoformat(),
+        ),
+        parse_env_config=False,
+    )
+    export_jobs_stream = ExportCompletedJobsStream(tap_gladly)
+    before_row = {
+        "record": "data",
+        "parameters": {
+            "endAt": (pendulum.now() - datetime.timedelta(days=3)).isoformat()
+        },
+    }
+    within_row = {
+        "record": "data",
+        "parameters": {
+            "endAt": (pendulum.now() - datetime.timedelta(hours=30)).isoformat()
+        },
+    }
+    after_row = {
+        "record": "data",
+        "parameters": {"endAt": pendulum.now().isoformat()},
+    }
+    assert not export_jobs_stream.post_process(before_row, None)
+    assert not export_jobs_stream.post_process(after_row, None)
+    assert export_jobs_stream.post_process(within_row, None)
+
+
 def test_filter_by_content_type():
     tap_gladly = Tapgladly(
         config=dict(