Make sure we update the last ingested tenant id when switching tenants

packages/flare/bin/constants.py

@@ -17,7 +17,7 @@ class PasswordKeys(Enum):
 
 
 class CollectionKeys(Enum):
-    CURRENT_TENANT_ID = "current_tenant_id"
+    LAST_INGESTED_TENANT_ID = "last_ingested_tenant_id"
     START_DATE = "start_date"
     TIMESTAMP_LAST_FETCH = "timestamp_last_fetch"
 

packages/flare/bin/cron_job_ingest_events.py

@@ -42,7 +42,8 @@ def main(logger: Logger, app: client.Application) -> None:
     ingest_metadata_only = get_ingest_metadata_only(app=app)
 
     save_last_fetched(app=app)
-    events_fetchd_count = 0
+    save_last_ingested_tenant_id(app=app, tenant_id=tenant_id)
+    events_fetched_count = 0
     for event, next_token in fetch_feed(
         logger=logger,
         app=app,
@@ -52,14 +53,13 @@ def main(logger: Logger, app: client.Application) -> None:
     ):
         save_last_fetched(app=app)
 
-        save_start_date(app=app, tenant_id=tenant_id)
         save_next(app=app, tenant_id=tenant_id, next=next_token)
 
         print(json.dumps(event), flush=True)
 
-        events_fetchd_count += 1
+        events_fetched_count += 1
 
-    logger.info(f"Fetchd {events_fetchd_count} events")
+    logger.info(f"Fetched {events_fetched_count} events")
 
 
 def get_storage_password_value(
@@ -120,12 +120,12 @@ def get_start_date(app: client.Application) -> Optional[date]:
     return None
 
 
-def get_current_tenant_id(app: client.Application) -> Optional[int]:
-    current_tenant_id = get_collection_value(
-        app=app, key=CollectionKeys.CURRENT_TENANT_ID.value
+def get_last_ingested_tenant_id(app: client.Application) -> Optional[int]:
+    last_ingested_tenant_id = get_collection_value(
+        app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value
     )
     try:
-        return int(current_tenant_id) if current_tenant_id else None
+        return int(last_ingested_tenant_id) if last_ingested_tenant_id else None
     except Exception:
         pass
     return None
@@ -151,24 +151,22 @@ def create_collection(app: client.Application) -> None:
         )
 
 
-def save_start_date(app: client.Application, tenant_id: int) -> None:
-    current_tenant_id = get_current_tenant_id(app=app)
-    # If this is the first request ever, insert today's date so that future requests will be based on that
-    if not get_start_date(app):
+def save_last_ingested_tenant_id(app: client.Application, tenant_id: int) -> None:
+    # If the tenant has changed, update the start date so that future requests will be based off today
+    # If you switch tenants, this will avoid the old tenant from ingesting all the events before today and the day
+    # that tenant was switched in the first place.
+    if get_last_ingested_tenant_id(app=app) != tenant_id:
         save_collection_value(
             app=app,
             key=CollectionKeys.START_DATE.value,
             value=date.today().isoformat(),
         )
 
-    # If the current tenant has changed, update the start date so that future requests will be based off today
-    # If you switch tenants, this will avoid the old tenant from ingesting all the events before today and the day
-    # that tenant was switched in the first place.
-    if current_tenant_id != tenant_id:
-        app.service.kvstore[KV_COLLECTION_NAME].data.update(
-            id=CollectionKeys.START_DATE.value,
-            data=json.dumps({"value": date.today().isoformat()}),
-        )
+    save_collection_value(
+        app=app,
+        key=CollectionKeys.LAST_INGESTED_TENANT_ID.value,
+        value=tenant_id,
+    )
 
 
 def save_next(app: client.Application, tenant_id: int, next: Optional[str]) -> None:

packages/flare/tests/bin/test_ingest_events.py

@@ -10,6 +10,7 @@
 from unittest.mock import MagicMock
 from unittest.mock import Mock
 from unittest.mock import PropertyMock
+from unittest.mock import call
 from unittest.mock import patch
 
 
@@ -20,13 +21,13 @@
 from cron_job_ingest_events import fetch_feed
 from cron_job_ingest_events import get_api_key
 from cron_job_ingest_events import get_collection_value
-from cron_job_ingest_events import get_current_tenant_id
 from cron_job_ingest_events import get_last_fetched
+from cron_job_ingest_events import get_last_ingested_tenant_id
 from cron_job_ingest_events import get_start_date
 from cron_job_ingest_events import get_tenant_id
 from cron_job_ingest_events import main
 from cron_job_ingest_events import save_collection_value
-from cron_job_ingest_events import save_start_date
+from cron_job_ingest_events import save_last_ingested_tenant_id
 
 
 def test_get_collection_value_expect_none() -> None:
@@ -121,19 +122,19 @@ def test_get_start_date_expect_date(get_collection_value_mock: MagicMock) -> Non
 
 
 @patch("cron_job_ingest_events.get_collection_value", return_value="not_a_number")
-def test_get_current_tenant_id_expect_none(
+def test_get_last_ingested_tenant_id_expect_none(
     get_collection_value_mock: MagicMock,
 ) -> None:
     app = MagicMock()
-    assert get_current_tenant_id(app=app) is None
+    assert get_last_ingested_tenant_id(app=app) is None
 
 
 @patch("cron_job_ingest_events.get_collection_value", return_value="11111")
-def test_get_current_tenant_id_expect_integer(
+def test_get_last_ingested_tenant_id_expect_integer(
     get_collection_value_mock: MagicMock,
 ) -> None:
     app = MagicMock()
-    assert get_current_tenant_id(app=app) == 11111
+    assert get_last_ingested_tenant_id(app=app) == 11111
 
 
 @patch(
@@ -154,35 +155,63 @@ def test_get_last_fetched_expect_datetime(get_collection_value_mock: MagicMock)
 
 
 @patch("cron_job_ingest_events.save_collection_value")
-@patch("cron_job_ingest_events.get_start_date", return_value=None)
-@patch("cron_job_ingest_events.get_current_tenant_id", return_value=11111)
-def test_save_start_date_expect_save_collection_value_called_and_tenant_id_unchanged(
-    get_current_tenant_id_mock: MagicMock,
+@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=None)
+def test_save_last_ingested_tenant_id_expect_save_collection_value_called_and_tenant_id_unchanged(
+    get_last_ingested_tenant_id_mock: MagicMock,
+    save_collection_value_mock: MagicMock,
+) -> None:
+    app = MagicMock()
+    save_last_ingested_tenant_id(app=app, tenant_id=11111)
+    save_collection_value_mock.assert_has_calls(
+        [
+            call(
+                app=app,
+                key=CollectionKeys.START_DATE.value,
+                value=date.today().isoformat(),
+            ),
+            call(
+                app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111
+            ),
+        ]
+    )
+
+
+@patch("cron_job_ingest_events.save_collection_value")
+@patch("cron_job_ingest_events.get_start_date", return_value=date.today())
+@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=22222)
+def test_save_last_ingested_tenant_id_expect_save_collection_value_not_called_and_tenant_id_changed(
+    get_last_ingested_tenant_id_mock: MagicMock,
     get_start_date_mock: MagicMock,
     save_collection_value_mock: MagicMock,
 ) -> None:
     app = MagicMock()
-    save_start_date(app=app, tenant_id=11111)
-    save_collection_value_mock.assert_called_once_with(
-        app=app, key=CollectionKeys.START_DATE.value, value=date.today().isoformat()
+    save_last_ingested_tenant_id(app=app, tenant_id=11111)
+    save_collection_value_mock.assert_has_calls(
+        [
+            call(
+                app=app,
+                key=CollectionKeys.START_DATE.value,
+                value=date.today().isoformat(),
+            ),
+            call(
+                app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111
+            ),
+        ]
     )
-    app.service.kvstore[KV_COLLECTION_NAME].data.update.assert_not_called()
 
 
 @patch("cron_job_ingest_events.save_collection_value")
 @patch("cron_job_ingest_events.get_start_date", return_value=date.today())
-@patch("cron_job_ingest_events.get_current_tenant_id", return_value=22222)
-def test_save_start_date_expect_save_collection_value_not_called_and_tenant_id_changed(
-    get_current_tenant_id_mock: MagicMock,
+@patch("cron_job_ingest_events.get_last_ingested_tenant_id", return_value=11111)
+def test_save_last_ingested_tenant_id_expect_same_tenant_id(
+    get_last_ingested_tenant_id_mock: MagicMock,
     get_start_date_mock: MagicMock,
     save_collection_value_mock: MagicMock,
 ) -> None:
     app = MagicMock()
-    save_start_date(app=app, tenant_id=11111)
-    save_collection_value_mock.assert_not_called()
-    app.service.kvstore[KV_COLLECTION_NAME].data.update.assert_called_once_with(
-        id=CollectionKeys.START_DATE.value,
-        data=json.dumps({"value": date.today().isoformat()}),
+    save_last_ingested_tenant_id(app=app, tenant_id=11111)
+    save_collection_value_mock.assert_called_once_with(
+        app=app, key=CollectionKeys.LAST_INGESTED_TENANT_ID.value, value=11111
     )
 
 

packages/react-components/src/StatusScreen.tsx

@@ -1,22 +1,22 @@
-import React, { useEffect, FC, useState } from 'react';
+import React, { FC, useEffect, useState } from 'react';
+import Button from './components/Button';
 import './global.css';
+import { SplunkCollectionItem } from './models/splunk';
 import './StatusScreen.css';
 import {
-    fetchTenantId,
-    fetchVersionName,
     fetchCollectionItems,
     fetchCurrentIndexName,
+    fetchTenantId,
+    fetchVersionName,
 } from './utils/setupConfiguration';
-import { SplunkCollectionItem } from './models/splunk';
-import Button from './components/Button';
 
 const COLLECTION_KEYS_NEXT_PREFIX = 'next_';
 
 enum StatusItemKeys {
     START_DATE = 'start_date',
     LAST_FETCHED = 'timestamp_last_fetch',
     NEXT_TOKEN = 'next_token',
-    CURRENT_TENANT_ID = 'current_tenant_id',
+    LAST_INGESTED_TENANT_ID = 'last_ingested_tenant_id',
     INDEX = 'index',
     VERSION = 'version',
 }
@@ -53,8 +53,8 @@ const StatusScreen: FC<{ theme: string }> = ({ theme }) => {
             ];
             const advancedItems: StatusItem[] = [
                 {
-                    key: StatusItemKeys.CURRENT_TENANT_ID,
-                    name: getItemName(StatusItemKeys.CURRENT_TENANT_ID),
+                    key: StatusItemKeys.LAST_INGESTED_TENANT_ID,
+                    name: getItemName(StatusItemKeys.LAST_INGESTED_TENANT_ID),
                     value: `${id}`,
                 },
             ];
@@ -104,8 +104,8 @@ const StatusScreen: FC<{ theme: string }> = ({ theme }) => {
             return 'Last moment the events were ingested';
         }
 
-        if (key === StatusItemKeys.CURRENT_TENANT_ID) {
-            return 'Current Tenant ID';
+        if (key === StatusItemKeys.LAST_INGESTED_TENANT_ID) {
+            return 'Last Tenant ID Ingested';
         }
 
         if (key === StatusItemKeys.INDEX) {