From 2a0fefd6e8fd86e8f877dcb7f52a41423c6061fc Mon Sep 17 00:00:00 2001 From: Tom Hodson Date: Tue, 21 Jan 2025 11:11:53 +0000 Subject: [PATCH] bit of extra logging --- cron_ingest.sh | 2 +- src/ionbeam/sources/API_sources_base.py | 4 ++-- src/ionbeam/sources/smart_citizen_kit/source.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cron_ingest.sh b/cron_ingest.sh index b7cea02..54f2dfa 100755 --- a/cron_ingest.sh +++ b/cron_ingest.sh @@ -2,7 +2,7 @@ cd /home/math/IonBeam export ODC_ENABLE_WRITING_LONG_STRING_CODEC=1 echo >> cron_logs_ingest.txt -echo Meteotracker `date` >> cron_logs_ingest.txt +echo Ingestion `date` >> cron_logs_ingest.txt /home/math/.venv/bin/python -m ionbeam \ ./config -vvv \ --env ewc \ diff --git a/src/ionbeam/sources/API_sources_base.py b/src/ionbeam/sources/API_sources_base.py index 0638176..2eec7bc 100644 --- a/src/ionbeam/sources/API_sources_base.py +++ b/src/ionbeam/sources/API_sources_base.py @@ -55,8 +55,8 @@ def __init__(self, retry_at: datetime | None = None, else: - logger.debug("No Retry-After header, delaying for one minute.") - self.retry_at = datetime.now(UTC) + timedelta(minutes=1) + logger.debug("No Retry-After header, delaying for 5 minutes.") + self.retry_at = datetime.now(UTC) + timedelta(minutes=5) @dataclasses.dataclass(eq=True, frozen=True) class DataChunk: diff --git a/src/ionbeam/sources/smart_citizen_kit/source.py b/src/ionbeam/sources/smart_citizen_kit/source.py index 82010ca..307c751 100644 --- a/src/ionbeam/sources/smart_citizen_kit/source.py +++ b/src/ionbeam/sources/smart_citizen_kit/source.py @@ -16,6 +16,7 @@ from ...core.bases import Mappings, RawVariable, TabularMessage, TimeSpan from ..API_sources_base import DataChunk, DataStream, RESTSource from ...singleprocess_pipeline import fmt_time +from time import sleep logger = logging.getLogger(__name__) @@ -214,6 +215,9 @@ def download_chunk(self, data_stream: DataStream, time_span: TimeSpan) -> DataCh readings = self.get_readings(device_id, sensor["id"], time_span) + # Try to reduce how often we get rate limited by SCK + sleep(0.5) + # Skip if there are now readings if not readings["readings"]: logger.debug(f"No readings returned for {sensor['name']}, even though the date metadata suggested there should be.")