From 84bf9743adfbab1d33cdcf5b90893add474020d5 Mon Sep 17 00:00:00 2001 From: Kristian Tashkov Date: Wed, 6 Sep 2023 01:00:31 +0300 Subject: [PATCH] :sparkles: Night owl service for completing daily tasks after midnight --- tools_for_todoist/app.py | 17 ++ tools_for_todoist/models/item.py | 2 +- tools_for_todoist/models/todoist.py | 12 +- .../services/night_owl_enabler.py | 55 +++++ tools_for_todoist/storage/__init__.py | 28 +-- tools_for_todoist/tests/mocks.py | 62 ++++++ .../tests/services/test_night_owl_enabler.py | 197 ++++++++++++++++++ tools_for_todoist/utils.py | 5 +- 8 files changed, 354 insertions(+), 24 deletions(-) create mode 100644 tools_for_todoist/services/night_owl_enabler.py create mode 100644 tools_for_todoist/tests/mocks.py create mode 100644 tools_for_todoist/tests/services/test_night_owl_enabler.py diff --git a/tools_for_todoist/app.py b/tools_for_todoist/app.py index 56955d9..c4f6705 100644 --- a/tools_for_todoist/app.py +++ b/tools_for_todoist/app.py @@ -26,6 +26,20 @@ from tools_for_todoist.models.google_calendar import GoogleCalendar from tools_for_todoist.models.todoist import Todoist from tools_for_todoist.services.calendar_to_todoist import CalendarToTodoistService +from tools_for_todoist.services.night_owl_enabler import NightOwlEnabler +from tools_for_todoist.storage import set_storage +from tools_for_todoist.storage.storage import LocalKeyValueStorage, PostgresKeyValueStorage + +DEFAULT_STORAGE = os.path.join(os.path.dirname(__file__), 'storage', 'store.json') + + +def setup_storage() -> None: + database_config = os.environ.get('DATABASE_URL', None) + if database_config is not None: + storage = PostgresKeyValueStorage(database_config) + else: + storage = LocalKeyValueStorage(os.environ.get('FILE_STORE', DEFAULT_STORAGE)) + set_storage(storage) def setup_logger(logging_level=logging.DEBUG): @@ -51,6 +65,7 @@ def run_sync_service(logger): todoist = Todoist() google_calendar = GoogleCalendar() calendar_service = CalendarToTodoistService(todoist, google_calendar) + night_owl_enabler = NightOwlEnabler(todoist, google_calendar) logger.info('Started syncing service.') while True: @@ -61,10 +76,12 @@ def run_sync_service(logger): while should_keep_syncing: todoist_sync_result = todoist.sync() should_keep_syncing = calendar_service.on_todoist_sync(todoist_sync_result) + should_keep_syncing |= night_owl_enabler.on_todoist_sync(todoist_sync_result) time.sleep(10) def main(): + setup_storage() logger = setup_logger(os.environ.get('LOGGING_LEVEL', logging.DEBUG)) retry_count = 0 max_retries = 5 diff --git a/tools_for_todoist/models/item.py b/tools_for_todoist/models/item.py index c9aa498..ef212c8 100644 --- a/tools_for_todoist/models/item.py +++ b/tools_for_todoist/models/item.py @@ -46,7 +46,7 @@ def raw(self): return self._raw @staticmethod - def from_raw(todoist, raw): + def from_raw(todoist, raw) -> 'TodoistItem': item = TodoistItem(todoist, raw['content'], raw['project_id']) item.id = raw['id'] item.update_from_raw(raw) diff --git a/tools_for_todoist/models/todoist.py b/tools_for_todoist/models/todoist.py index d0c5da5..a3cb683 100644 --- a/tools_for_todoist/models/todoist.py +++ b/tools_for_todoist/models/todoist.py @@ -17,6 +17,9 @@ with this program. If not, see . """ import logging +from contextlib import ExitStack +from tempfile import TemporaryDirectory +from typing import Optional from requests import Session from todoist.api import SyncError, TodoistAPI @@ -32,6 +35,7 @@ class Todoist: def __init__(self): + self._exit_stack: Optional[ExitStack] = None self._recreate_api() self.api.reset_state() self._items = {} @@ -40,11 +44,15 @@ def __init__(self): self._initial_sync() def _recreate_api(self): + if self._exit_stack is not None: + self._exit_stack.close() + self._exit_stack = ExitStack() token = get_storage().get_value(TODOIST_API_KEY) headered_session = Session() headered_session.headers['Authorization'] = f'Bearer {token}' - self.api = TodoistAPI(token, session=headered_session, api_version='v9') + new_temp_dir = self._exit_stack.enter_context(TemporaryDirectory()) + self.api = TodoistAPI(token, session=headered_session, api_version='v9', cache=new_temp_dir) def _activity_sync(self, offset=0, limit=100): def activity_get_func(): @@ -124,7 +132,7 @@ def _update_items(self, raw_updated_items): updated_items.append((old_item, item_model)) return {'deleted': deleted_items, 'created': new_items, 'updated': updated_items} - def get_item_by_id(self, item_id): + def get_item_by_id(self, item_id: str) -> TodoistItem: return self._items.get(item_id) def get_project_by_name(self, name): diff --git a/tools_for_todoist/services/night_owl_enabler.py b/tools_for_todoist/services/night_owl_enabler.py new file mode 100644 index 0000000..3cbea09 --- /dev/null +++ b/tools_for_todoist/services/night_owl_enabler.py @@ -0,0 +1,55 @@ +""" +Copyright (C) 2020-2023 Kristian Tashkov + +This file is part of "Tools for Todoist". + +"Tools for Todoist" is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your +option) any later version. + +"Tools for Todoist" is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with this program. If not, see . +""" +from datetime import datetime +from typing import Any, Dict + +from dateutil.tz import gettz + +from tools_for_todoist.models.google_calendar import GoogleCalendar +from tools_for_todoist.models.todoist import Todoist +from tools_for_todoist.storage import get_storage + +NIGHT_OWL_DAY_SWITCH_HOUR = 'night_owl.day_switch_hour' + + +class NightOwlEnabler: + def __init__(self, todoist: Todoist, calendar: GoogleCalendar) -> None: + self._todoist = todoist + self._google_calendar = calendar + self._day_switch_hour = int(get_storage().get_value(NIGHT_OWL_DAY_SWITCH_HOUR, 4)) + + def on_todoist_sync(self, sync_result: Dict[str, Any]) -> bool: + should_sync = False + for item_id in sync_result['completed']: + item = self._todoist.get_item_by_id(item_id) + if ( + item is None + or item.get_due_string() is None + or 'every day' not in item.get_due_string() + ): + continue + now = datetime.now(gettz(self._google_calendar.default_timezone)) + seconds_from_midnight = (now - now.replace(hour=0, minute=0, second=0)).total_seconds() + if seconds_from_midnight / 3600 > self._day_switch_hour: + continue + + next_due = item.next_due_date().replace(year=now.year, month=now.month, day=now.day) + item.set_due(next_date=next_due, due_string=item.get_due_string()) + should_sync |= item.save() + return should_sync diff --git a/tools_for_todoist/storage/__init__.py b/tools_for_todoist/storage/__init__.py index 2853a31..2e14903 100644 --- a/tools_for_todoist/storage/__init__.py +++ b/tools_for_todoist/storage/__init__.py @@ -16,31 +16,19 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ -import os +from typing import Optional -from tools_for_todoist.storage.storage import LocalKeyValueStorage, PostgresKeyValueStorage +from tools_for_todoist.storage.storage import KeyValueStorage -DEFAULT_STORAGE = os.path.join(os.path.dirname(__file__), 'store.json') -_storage = None +_storage: Optional[KeyValueStorage] = None -def get_storage(): +def set_storage(storage: KeyValueStorage) -> None: global _storage - if _storage is not None: - return _storage + _storage = storage - database_config = os.environ.get('DATABASE_URL', None) - if database_config is not None: - _storage = PostgresKeyValueStorage(database_config) - else: - _storage = LocalKeyValueStorage(os.environ.get('FILE_STORE', DEFAULT_STORAGE)) - return _storage - - -def reinitialize_storage(): +def get_storage() -> KeyValueStorage: global _storage - if _storage is not None: - _storage.close() - _storage = None - return get_storage() + assert _storage is not None + return _storage diff --git a/tools_for_todoist/tests/mocks.py b/tools_for_todoist/tests/mocks.py new file mode 100644 index 0000000..775f905 --- /dev/null +++ b/tools_for_todoist/tests/mocks.py @@ -0,0 +1,62 @@ +""" +Copyright (C) 2020-2023 Kristian Tashkov + +This file is part of "Tools for Todoist". + +"Tools for Todoist" is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your +option) any later version. + +"Tools for Todoist" is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with this program. If not, see . +""" +from contextlib import ExitStack +from typing import Any, Dict, Optional +from unittest import TestCase +from unittest.mock import MagicMock + +from tools_for_todoist.models.item import TodoistItem +from tools_for_todoist.storage import set_storage +from tools_for_todoist.storage.storage import KeyValueStorage + + +class ServicesTestCase(TestCase): + def setUp(self) -> None: + self._exit_stack = ExitStack() + + self._todoist_items: Dict[str, TodoistItem] = {} + self._todoist_mock = MagicMock() + self._todoist_mock.get_item_by_id.side_effect = lambda item_id: self._todoist_items.get( + item_id + ) + + self._google_calendar_mock = MagicMock() + self._google_calendar_mock.default_timezone = 'Europe/Zurich' + + self._storage = KeyValueStorage() + set_storage(self._storage) + + def tearDown(self) -> None: + self._exit_stack.close() + + def _create_todoist_item(self, due: Optional[Dict[str, Any]] = None) -> TodoistItem: + raw_item = { + 'id': 'TEST_ITEM_ID', + 'project_id': 'TEST_PROJECT_ID', + 'content': 'Test item', + 'priority': 1, + 'description': 'Test item description', + 'checked': False, + 'duration': None, + 'labels': [], + 'due': due, + } + item = TodoistItem.from_raw(self._todoist_mock, raw_item) + self._todoist_items[item.id] = item + return item diff --git a/tools_for_todoist/tests/services/test_night_owl_enabler.py b/tools_for_todoist/tests/services/test_night_owl_enabler.py new file mode 100644 index 0000000..487062f --- /dev/null +++ b/tools_for_todoist/tests/services/test_night_owl_enabler.py @@ -0,0 +1,197 @@ +""" +Copyright (C) 2020-2023 Kristian Tashkov + +This file is part of "Tools for Todoist". + +"Tools for Todoist" is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your +option) any later version. + +"Tools for Todoist" is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with this program. If not, see . +""" +from datetime import datetime +from unittest.mock import patch + +from tools_for_todoist.services.night_owl_enabler import NIGHT_OWL_DAY_SWITCH_HOUR, NightOwlEnabler +from tools_for_todoist.tests.mocks import ServicesTestCase + + +class NightOwlEnablerTests(ServicesTestCase): + def setUp(self): + super().setUp() + self._storage.set_value(NIGHT_OWL_DAY_SWITCH_HOUR, 2) + + def _set_current_time(self, day: int, hour: int) -> None: + datetime_mock = self._exit_stack.enter_context( + patch( + 'tools_for_todoist.services.night_owl_enabler.datetime', + ) + ) + datetime_mock.now.return_value = datetime(year=2020, month=1, day=day, hour=hour) + + def test_empty_completed(self) -> None: + self._create_todoist_item( + due={ + 'date': '2020-01-10T23:00:00', + 'string': 'every day at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + night_owl_enabler.on_todoist_sync({'completed': []}) + self.assertFalse(self._todoist_mock.update_item.called) + + def test_completed_before_midnight(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10T23:00:00', + 'string': 'every day at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=10, hour=23) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_not_called() + + def test_completed_before_midnight_date_only(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10', + 'string': 'every day', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=10, hour=23) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_not_called() + + def test_completed_non_recurring(self) -> None: + test_item = self._create_todoist_item() + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=11, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_not_called() + + def test_completed_non_every_day(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10T23:00:00', + 'string': 'every 23 hour at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=11, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_not_called() + + def test_completed_after_midnight(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10T23:00:00', + 'string': 'every day at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=11, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_called_once_with( + test_item, + due={ + 'date': '2020-01-11T23:00:00', + 'string': 'every day at 23:00', + 'timezone': None, + }, + ) + + def test_completed_after_midnight_date_only(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10', + 'string': 'every day', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=11, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_called_once_with( + test_item, + due={ + 'date': '2020-01-11', + 'string': 'every day', + 'timezone': None, + }, + ) + + def test_completed_after_midnight_date_overdue(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10', + 'string': 'every day', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=14, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_called_once_with( + test_item, + due={ + 'date': '2020-01-14', + 'string': 'every day', + 'timezone': None, + }, + ) + + def test_completed_after_midnight_overdue(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10T23:00:00', + 'string': 'every day at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=15, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_called_once_with( + test_item, + due={ + 'date': '2020-01-15T23:00:00', + 'string': 'every day at 23:00', + 'timezone': None, + }, + ) + + def test_completed_after_midnight_timezone(self) -> None: + test_item = self._create_todoist_item( + due={ + 'date': '2020-01-10T20:00:00Z', + 'timezone': 'Europe/Sofia', + 'string': 'every day at 23:00', + } + ) + night_owl_enabler = NightOwlEnabler(self._todoist_mock, self._google_calendar_mock) + + self._set_current_time(day=11, hour=1) + night_owl_enabler.on_todoist_sync({'completed': [test_item.id]}) + self._todoist_mock.update_item.assert_called_once_with( + test_item, + due={ + 'date': '2020-01-11T20:00:00Z', + 'string': 'every day at 23:00', + 'timezone': 'Europe/Sofia', + }, + ) diff --git a/tools_for_todoist/utils.py b/tools_for_todoist/utils.py index 35c78bc..2b0a799 100644 --- a/tools_for_todoist/utils.py +++ b/tools_for_todoist/utils.py @@ -24,6 +24,8 @@ from dateutil.tz import UTC +from tools_for_todoist.storage import get_storage + logger = logging.getLogger(__name__) @@ -61,7 +63,8 @@ def to_todoist_date(dt): def retry_flaky_function( func, name, validate_result_func=None, on_failure_func=None, critical_errors=None ): - for attempt in range(1, 6): + retry_count = get_storage().get_value('global.retry_count', 5) + for attempt in range(1, retry_count + 1): try: result = func() if validate_result_func is not None and not validate_result_func(result):