diff --git a/README.md b/README.md index 1e45593..b62df6d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tap-gladly -`tap-gladly` is a Singer tap for gladly. +`tap-gladly` is a Singer tap for Gladly and the Export API Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps. @@ -40,7 +40,8 @@ pipx install git+https://github.com/ORG_NAME/tap-gladly.git@main | username | True | None | The username to authenticate against the API service | | password | True | None | The username to authenticate against the API service | | project_ids | False | None | Project IDs to replicate | -| start_date | True | None | The earliest record date to sync, format %Y-%m-%dT%H:%M:%SZ | +| start_date | True | None | The earliest job date to sync, format %Y-%m-%dT%H:%M:%SZ | +| end_date | False | None | The latest job date to sync, format %Y-%m-%dT%H:%M:%SZ | | api_url_base | True | None | The url for the API service | | stream_maps | False | None | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). | | stream_map_config | False | None | User-defined config values to be used within map expressions. | diff --git a/pyproject.toml b/pyproject.toml index 858a1d4..d1439fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-gladly" -version = "0.2.0" +version = "0.3.0" description = "`tap-gladly` is a Singer tap for gladly, built with the Meltano SDK for Singer Taps." authors = ["harrystech"] keywords = [ diff --git a/tap_gladly/streams.py b/tap_gladly/streams.py index 8a57c20..2af6f34 100644 --- a/tap_gladly/streams.py +++ b/tap_gladly/streams.py @@ -26,10 +26,13 @@ class ExportCompletedJobsStream(gladlyStream): def post_process(self, row, context): """Filter jobs that finished before start_date.""" - if pendulum.parse(row["parameters"]["endAt"]) >= pendulum.parse( - self.config["start_date"] - ): - return row + job_completion_date = pendulum.parse(row["parameters"]["endAt"]) + if pendulum.parse(self.config["start_date"]) <= job_completion_date: + if "end_date" in self.config: + if pendulum.parse(self.config["end_date"]) >= job_completion_date: + return row + else: + return row return def get_child_context(self, record: dict, context: Optional[dict]) -> dict: diff --git a/tap_gladly/tap.py b/tap_gladly/tap.py index 9ef3819..f83fc89 100644 --- a/tap_gladly/tap.py +++ b/tap_gladly/tap.py @@ -69,7 +69,12 @@ class Tapgladly(Tap): "start_date", th.DateTimeType, required=True, - description="The earliest record date to sync, format %Y-%m-%dT%H:%M:%SZ", + description="The earliest job date to sync, format %Y-%m-%dT%H:%M:%SZ", + ), + th.Property( + "end_date", + th.DateTimeType, + description="The latest job date to sync, format %Y-%m-%dT%H:%M:%SZ", ), th.Property( "api_url_base", diff --git a/tap_gladly/tests/test_streams.py b/tap_gladly/tests/test_streams.py index 801fa21..6c0990b 100644 --- a/tap_gladly/tests/test_streams.py +++ b/tap_gladly/tests/test_streams.py @@ -41,6 +41,37 @@ def test_started_at(): assert export_jobs_stream.post_process(after_row, None) +def test_data_interval(): + tap_gladly = Tapgladly( + config=dict( + SAMPLE_CONFIG, + start_date=(pendulum.now() - datetime.timedelta(days=2)).isoformat(), + end_date=(pendulum.now() - datetime.timedelta(days=1)).isoformat(), + ), + parse_env_config=False, + ) + export_jobs_stream = ExportCompletedJobsStream(tap_gladly) + before_row = { + "record": "data", + "parameters": { + "endAt": (pendulum.now() - datetime.timedelta(days=3)).isoformat() + }, + } + within_row = { + "record": "data", + "parameters": { + "endAt": (pendulum.now() - datetime.timedelta(hours=30)).isoformat() + }, + } + after_row = { + "record": "data", + "parameters": {"endAt": pendulum.now().isoformat()}, + } + assert not export_jobs_stream.post_process(before_row, None) + assert not export_jobs_stream.post_process(after_row, None) + assert export_jobs_stream.post_process(within_row, None) + + def test_filter_by_content_type(): tap_gladly = Tapgladly( config=dict(