diff --git a/packages/flare/bin/constants.py b/packages/flare/bin/constants.py index b5e1d02..6374a9b 100644 --- a/packages/flare/bin/constants.py +++ b/packages/flare/bin/constants.py @@ -17,7 +17,7 @@ class PasswordKeys(Enum): class CollectionKeys(Enum): - CURRENT_TENANT_ID = "current_tenant_id" + LAST_INGESTED_TENANT_ID = "last_ingested_tenant_id" START_DATE = "start_date" TIMESTAMP_LAST_FETCH = "timestamp_last_fetch" diff --git a/packages/flare/bin/cron_job_ingest_events.py b/packages/flare/bin/cron_job_ingest_events.py index 700e5db..e50ceb6 100644 --- a/packages/flare/bin/cron_job_ingest_events.py +++ b/packages/flare/bin/cron_job_ingest_events.py @@ -42,7 +42,8 @@ def main(logger: Logger, app: client.Application) -> None: ingest_metadata_only = get_ingest_metadata_only(app=app) save_last_fetched(app=app) - events_fetchd_count = 0 + save_last_ingested_tenant_id(app=app, tenant_id=tenant_id) + events_fetched_count = 0 for event, next_token in fetch_feed( logger=logger, app=app, @@ -52,14 +53,13 @@ def main(logger: Logger, app: client.Application) -> None: ): save_last_fetched(app=app) - save_start_date(app=app, tenant_id=tenant_id) save_next(app=app, tenant_id=tenant_id, next=next_token) print(json.dumps(event), flush=True) - events_fetchd_count += 1 + events_fetched_count += 1 - logger.info(f"Fetchd {events_fetchd_count} events") + logger.info(f"Fetched {events_fetched_count} events") def get_storage_password_value( @@ -120,12 +120,12 @@ def get_start_date(app: client.Application) -> Optional[date]: return None -def get_current_tenant_id(app: client.Application) -> Optional[int]: - current_tenant_id = get_collection_value( - app=app, key=CollectionKeys.CURRENT_TENANT_ID.value +def get_last_ingested_tenant_id(app: client.Application) -> Optional[int]: + last_ingested_tenant_id = get_collection_value( + app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value ) try: - return int(current_tenant_id) if current_tenant_id else None + return int(last_ingested_tenant_id) if last_ingested_tenant_id else None except Exception: pass return None @@ -151,24 +151,22 @@ def create_collection(app: client.Application) -> None: ) -def save_start_date(app: client.Application, tenant_id: int) -> None: - current_tenant_id = get_current_tenant_id(app=app) - # If this is the first request ever, insert today's date so that future requests will be based on that - if not get_start_date(app): +def save_last_ingested_tenant_id(app: client.Application, tenant_id: int) -> None: + # If the tenant has changed, update the start date so that future requests will be based off today + # If you switch tenants, this will avoid the old tenant from ingesting all the events before today and the day + # that tenant was switched in the first place. + if get_last_ingested_tenant_id(app=app) != tenant_id: save_collection_value( app=app, key=CollectionKeys.START_DATE.value, value=date.today().isoformat(), ) - # If the current tenant has changed, update the start date so that future requests will be based off today - # If you switch tenants, this will avoid the old tenant from ingesting all the events before today and the day - # that tenant was switched in the first place. - if current_tenant_id != tenant_id: - app.service.kvstore[KV_COLLECTION_NAME].data.update( - id=CollectionKeys.START_DATE.value, - data=json.dumps({"value": date.today().isoformat()}), - ) + save_collection_value( + app=app, + key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, + value=tenant_id, + ) def save_next(app: client.Application, tenant_id: int, next: Optional[str]) -> None: diff --git a/packages/flare/tests/bin/test_ingest_events.py b/packages/flare/tests/bin/test_ingest_events.py index 1b7b26b..c836bba 100644 --- a/packages/flare/tests/bin/test_ingest_events.py +++ b/packages/flare/tests/bin/test_ingest_events.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock from unittest.mock import Mock from unittest.mock import PropertyMock +from unittest.mock import call from unittest.mock import patch @@ -20,13 +21,13 @@ from cron_job_ingest_events import fetch_feed from cron_job_ingest_events import get_api_key from cron_job_ingest_events import get_collection_value -from cron_job_ingest_events import get_current_tenant_id from cron_job_ingest_events import get_last_fetched +from cron_job_ingest_events import get_last_ingested_tenant_id from cron_job_ingest_events import get_start_date from cron_job_ingest_events import get_tenant_id from cron_job_ingest_events import main from cron_job_ingest_events import save_collection_value -from cron_job_ingest_events import save_start_date +from cron_job_ingest_events import save_last_ingested_tenant_id def test_get_collection_value_expect_none() -> None: @@ -121,19 +122,19 @@ def test_get_start_date_expect_date(get_collection_value_mock: MagicMock) -> Non @patch("cron_job_ingest_events.get_collection_value", return_value="not_a_number") -def test_get_current_tenant_id_expect_none( +def test_get_last_ingested_tenant_id_expect_none( get_collection_value_mock: MagicMock, ) -> None: app = MagicMock() - assert get_current_tenant_id(app=app) is None + assert get_last_ingested_tenant_id(app=app) is None @patch("cron_job_ingest_events.get_collection_value", return_value="11111") -def test_get_current_tenant_id_expect_integer( +def test_get_last_ingested_tenant_id_expect_integer( get_collection_value_mock: MagicMock, ) -> None: app = MagicMock() - assert get_current_tenant_id(app=app) == 11111 + assert get_last_ingested_tenant_id(app=app) == 11111 @patch( @@ -154,35 +155,63 @@ def test_get_last_fetched_expect_datetime(get_collection_value_mock: MagicMock) @patch("cron_job_ingest_events.save_collection_value") -@patch("cron_job_ingest_events.get_start_date", return_value=None) -@patch("cron_job_ingest_events.get_current_tenant_id", return_value=11111) -def test_save_start_date_expect_save_collection_value_called_and_tenant_id_unchanged( - get_current_tenant_id_mock: MagicMock, +@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=None) +def test_save_last_ingested_tenant_id_expect_save_collection_value_called_and_tenant_id_unchanged( + get_last_ingested_tenant_id_mock: MagicMock, + save_collection_value_mock: MagicMock, +) -> None: + app = MagicMock() + save_last_ingested_tenant_id(app=app, tenant_id=11111) + save_collection_value_mock.assert_has_calls( + [ + call( + app=app, + key=CollectionKeys.START_DATE.value, + value=date.today().isoformat(), + ), + call( + app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111 + ), + ] + ) + + +@patch("cron_job_ingest_events.save_collection_value") +@patch("cron_job_ingest_events.get_start_date", return_value=date.today()) +@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=22222) +def test_save_last_ingested_tenant_id_expect_save_collection_value_not_called_and_tenant_id_changed( + get_last_ingested_tenant_id_mock: MagicMock, get_start_date_mock: MagicMock, save_collection_value_mock: MagicMock, ) -> None: app = MagicMock() - save_start_date(app=app, tenant_id=11111) - save_collection_value_mock.assert_called_once_with( - app=app, key=CollectionKeys.START_DATE.value, value=date.today().isoformat() + save_last_ingested_tenant_id(app=app, tenant_id=11111) + save_collection_value_mock.assert_has_calls( + [ + call( + app=app, + key=CollectionKeys.START_DATE.value, + value=date.today().isoformat(), + ), + call( + app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111 + ), + ] ) - app.service.kvstore[KV_COLLECTION_NAME].data.update.assert_not_called() @patch("cron_job_ingest_events.save_collection_value") @patch("cron_job_ingest_events.get_start_date", return_value=date.today()) -@patch("cron_job_ingest_events.get_current_tenant_id", return_value=22222) -def test_save_start_date_expect_save_collection_value_not_called_and_tenant_id_changed( - get_current_tenant_id_mock: MagicMock, +@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=11111) +def test_save_last_ingested_tenant_id_expect_same_tenant_id( + get_last_ingested_tenant_id_mock: MagicMock, get_start_date_mock: MagicMock, save_collection_value_mock: MagicMock, ) -> None: app = MagicMock() - save_start_date(app=app, tenant_id=11111) - save_collection_value_mock.assert_not_called() - app.service.kvstore[KV_COLLECTION_NAME].data.update.assert_called_once_with( - id=CollectionKeys.START_DATE.value, - data=json.dumps({"value": date.today().isoformat()}), + save_last_ingested_tenant_id(app=app, tenant_id=11111) + save_collection_value_mock.assert_called_once_with( + app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111 ) diff --git a/packages/react-components/src/StatusScreen.tsx b/packages/react-components/src/StatusScreen.tsx index 4a14d93..c05f1e5 100644 --- a/packages/react-components/src/StatusScreen.tsx +++ b/packages/react-components/src/StatusScreen.tsx @@ -1,14 +1,14 @@ -import React, { useEffect, FC, useState } from 'react'; +import React, { FC, useEffect, useState } from 'react'; +import Button from './components/Button'; import './global.css'; +import { SplunkCollectionItem } from './models/splunk'; import './StatusScreen.css'; import { - fetchTenantId, - fetchVersionName, fetchCollectionItems, fetchCurrentIndexName, + fetchTenantId, + fetchVersionName, } from './utils/setupConfiguration'; -import { SplunkCollectionItem } from './models/splunk'; -import Button from './components/Button'; const COLLECTION_KEYS_NEXT_PREFIX = 'next_'; @@ -16,7 +16,7 @@ enum StatusItemKeys { START_DATE = 'start_date', LAST_FETCHED = 'timestamp_last_fetch', NEXT_TOKEN = 'next_token', - CURRENT_TENANT_ID = 'current_tenant_id', + LAST_INGESTED_TENANT_ID = 'last_ingested_tenant_id', INDEX = 'index', VERSION = 'version', } @@ -53,8 +53,8 @@ const StatusScreen: FC<{ theme: string }> = ({ theme }) => { ]; const advancedItems: StatusItem[] = [ { - key: StatusItemKeys.CURRENT_TENANT_ID, - name: getItemName(StatusItemKeys.CURRENT_TENANT_ID), + key: StatusItemKeys.LAST_INGESTED_TENANT_ID, + name: getItemName(StatusItemKeys.LAST_INGESTED_TENANT_ID), value: `${id}`, }, ]; @@ -104,8 +104,8 @@ const StatusScreen: FC<{ theme: string }> = ({ theme }) => { return 'Last moment the events were ingested'; } - if (key === StatusItemKeys.CURRENT_TENANT_ID) { - return 'Current Tenant ID'; + if (key === StatusItemKeys.LAST_INGESTED_TENANT_ID) { + return 'Last Tenant ID Ingested'; } if (key === StatusItemKeys.INDEX) {