Severity filter

- Fetch available severities via the rest API
- Added picker in the user preference configuration step
- Unit tests
- Logic filtering is:
	- Minimum one severity must be selected
	- If all of the severities are selected, no filter is applied in case we add new severities in the future
	- Otherwise, pass the severities selected as a filter when fetching the events

packages/flare/bin/constants.py

@@ -14,6 +14,7 @@ class PasswordKeys(Enum):
     API_KEY = "api_key"
     TENANT_ID = "tenant_id"
     INGEST_METADATA_ONLY = "ingest_metadata_only"
+    SEVERITIES_FILTER = "severities_filter"
 
 
 class CollectionKeys(Enum):

packages/flare/bin/cron_job_ingest_events.py

@@ -93,6 +93,7 @@ def main(
     api_key = get_api_key(storage_passwords=storage_passwords)
     tenant_id = get_tenant_id(storage_passwords=storage_passwords)
     ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords)
+    severities_filter = get_severities_filter(storage_passwords=storage_passwords)
 
     save_last_fetched(kvstore=kvstore)
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=tenant_id)
@@ -103,6 +104,7 @@ def main(
         api_key=api_key,
         tenant_id=tenant_id,
         ingest_metadata_only=ingest_metadata_only,
+        severities=severities_filter,
     ):
         save_last_fetched(kvstore=kvstore)
 
@@ -158,6 +160,18 @@ def get_ingest_metadata_only(storage_passwords: StoragePasswords) -> bool:
     )
 
 
+def get_severities_filter(storage_passwords: StoragePasswords) -> list[str]:
+    severities_filter = get_storage_password_value(
+        storage_passwords=storage_passwords,
+        password_key=PasswordKeys.SEVERITIES_FILTER.value,
+    )
+
+    if severities_filter:
+        return severities_filter.split(",")
+
+    return []
+
+
 def get_next(kvstore: KVStoreCollections, tenant_id: int) -> Optional[str]:
     return get_collection_value(
         kvstore=kvstore, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}"
@@ -281,6 +295,7 @@ def fetch_feed(
     api_key: str,
     tenant_id: int,
     ingest_metadata_only: bool,
+    severities: list[str],
 ) -> Iterator[tuple[dict, str]]:
     try:
         flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id)
@@ -292,6 +307,7 @@ def fetch_feed(
             next=next,
             start_date=start_date,
             ingest_metadata_only=ingest_metadata_only,
+            severities=severities,
         ):
             yield event_next
     except Exception as e:

packages/flare/bin/flare.py

@@ -50,10 +50,12 @@ def fetch_feed_events(
         next: Optional[str] = None,
         start_date: Optional[date] = None,
         ingest_metadata_only: bool,
+        severities: list[str],
     ) -> Iterator[tuple[dict, str]]:
         for response in self._fetch_event_feed_metadata(
             next=next,
             start_date=start_date,
+            severities=severities,
         ):
             event_feed = response.json()
             self.logger.debug(event_feed)
@@ -71,6 +73,7 @@ def _fetch_event_feed_metadata(
         *,
         next: Optional[str] = None,
         start_date: Optional[date] = None,
+        severities: list[str],
     ) -> Iterator[requests.Response]:
         data: Dict[str, Any] = {
             "from": next if next else None,
@@ -79,10 +82,13 @@ def _fetch_event_feed_metadata(
                     "gte": start_date.isoformat()
                     if start_date
                     else date.today().isoformat()
-                }
+                },
             },
         }
 
+        if len(severities):
+            data["severity"] = severities
+
         for response in self.flare_client.scroll(
             method="POST",
             url="/firework/v4/events/tenant/_search",
@@ -107,3 +113,8 @@ def fetch_tenants(self) -> requests.Response:
         return self.flare_client.get(
             url="/firework/v2/me/tenants",
         )
+
+    def fetch_filters_severity(self) -> requests.Response:
+        return self.flare_client.get(
+            url="/firework/v4/events/filters/severities",
+        )

packages/flare/bin/flare_external_requests.py

@@ -41,3 +41,20 @@ def handle_POST(self) -> None:
         logger.debug(f"FlareUserTenants: {response_json}")
         self.response.setHeader("Content-Type", "application/json")
         self.response.write(json.dumps(response_json))
+
+
+class FlareSeverityFilters(splunk.rest.BaseRestHandler):
+    def handle_POST(self) -> None:
+        logger = Logger(class_name=__file__)
+        payload = self.request["payload"]
+        params = parse.parse_qs(payload)
+
+        if "apiKey" not in params:
+            raise Exception("API Key is required")
+
+        flare_api = FlareAPI(api_key=params["apiKey"][0])
+        response = flare_api.fetch_filters_severity()
+        response_json = response.json()
+        logger.debug(f"FlareSeverityFilters: {response_json}")
+        self.response.setHeader("Content-Type", "application/json")
+        self.response.write(json.dumps(response_json))

packages/flare/src/main/resources/splunk/default/restmap.conf

@@ -7,3 +7,8 @@ python.version = python3
 match=/fetch_user_tenants
 handler=flare_external_requests.FlareUserTenants
 python.version = python3
+
+[script:flare_external_requests_severity_filters]
+match=/fetch_severity_filters
+handler=flare_external_requests.FlareSeverityFilters
+python.version = python3

packages/flare/src/main/resources/splunk/default/web.conf

@@ -5,3 +5,7 @@ methods=POST
 [expose:flare_external_requests_user_tenants]
 pattern=fetch_user_tenants
 methods=POST
+
+[expose:flare_external_requests_severity_filters]
+pattern=fetch_severity_filters
+methods=POST

packages/flare/tests/bin/test_flare_wrapper.py

@@ -44,6 +44,7 @@ def test_flare_full_data_without_metadata(
         next=None,
         start_date=None,
         ingest_metadata_only=True,
+        severities=[],
     ):
         assert next_token == expected_return_value["next"]
         events.append(event)
@@ -105,6 +106,7 @@ def test_flare_full_data_with_metadata(
         next=None,
         start_date=None,
         ingest_metadata_only=False,
+        severities=[],
     ):
         assert next_token == expected_return_value["next"]
         events.append(event)
@@ -147,6 +149,7 @@ def test_flare_full_data_with_metadata_and_exception(
                 next=None,
                 start_date=None,
                 ingest_metadata_only=False,
+                severities=[],
             )
         )
 

packages/flare/tests/bin/test_ingest_events.py

@@ -228,6 +228,7 @@ def test_fetch_feed_expect_exception() -> None:
         api_key="some_key",
         tenant_id=11111,
         ingest_metadata_only=False,
+        severities=[],
     ):
         pass
 
@@ -262,6 +263,7 @@ def test_fetch_feed_expect_feed_response(
         api_key="some_key",
         tenant_id=11111,
         ingest_metadata_only=False,
+        severities=[],
     ):
         assert next_token == next
         events.append(event)
@@ -320,4 +322,5 @@ def test_main_expect_normal_run(
         api_key="some_api_key",
         tenant_id=111,
         ingest_metadata_only=False,
+        severities=[],
     )

packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx

@@ -1,20 +1,25 @@
 import React, { FC, useEffect, useState } from 'react';
-import { ConfigurationStep, Tenant } from '../models/flare';
+import { ConfigurationStep, Severity, Tenant } from '../models/flare';
 import Button from './Button';
 import Label from './Label';
 import Select from './Select';
 
 import { APP_NAME } from '../models/constants';
 import {
+    convertSeverityFilterToArray,
     fetchAvailableIndexNames,
     fetchCurrentIndexName,
+    fetchSeverityFilters,
     fetchIngestMetadataOnly,
+    fetchSeveritiesFilter,
     fetchTenantId,
     fetchUserTenants,
+    getSeverityFilterValue,
     saveConfiguration,
 } from '../utils/setupConfiguration';
 import './ConfigurationGlobalStep.css';
 import './ConfigurationUserPreferencesStep.css';
+import SeverityOptions from './SeverityOptions';
 import Switch from './Switch';
 import { ToastKeys, toastManager } from './ToastManager';
 import Tooltip from './Tooltip';
@@ -28,6 +33,8 @@ const ConfigurationUserPreferencesStep: FC<{
 }> = ({ show, configurationStep, apiKey, onNavigateBackClick, onUserPreferencesSaved }) => {
     const [tenantId, setTenantId] = useState<number | undefined>(undefined);
     const [tenants, setUserTenants] = useState<Tenant[]>([]);
+    const [selectedSeverities, setSelectedSeverities] = useState<Severity[]>([]);
+    const [severities, setSeverities] = useState<Severity[]>([]);
     const [indexName, setIndexName] = useState('');
     const [indexNames, setIndexNames] = useState<string[]>([]);
     const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false);
@@ -41,7 +48,13 @@ const ConfigurationUserPreferencesStep: FC<{
     const handleSubmitUserPreferences = (): void => {
         setIsLoading(true);
 
-        saveConfiguration(apiKey, Number(tenantId), indexName, isIngestingMetadataOnly)
+        saveConfiguration(
+            apiKey,
+            Number(tenantId),
+            indexName,
+            isIngestingMetadataOnly,
+            getSeverityFilterValue(selectedSeverities, severities)
+        )
             .then(() => {
                 setIsLoading(false);
                 toastManager.destroy(ToastKeys.ERROR);
@@ -69,17 +82,33 @@ const ConfigurationUserPreferencesStep: FC<{
                 fetchCurrentIndexName(),
                 fetchUserTenants(apiKey),
                 fetchAvailableIndexNames(),
+                fetchSeverityFilters(apiKey),
+                fetchSeveritiesFilter(),
             ])
-                .then(([id, ingestMetadataOnly, index, userTenants, availableIndexNames]) => {
-                    setTenantId(id);
-                    setIsIngestingMetadataOnly(ingestMetadataOnly);
-                    setIndexName(index);
-                    if (id === -1 && userTenants.length > 0) {
-                        setTenantId(userTenants[0].id);
+                .then(
+                    ([
+                        id,
+                        ingestMetadataOnly,
+                        index,
+                        userTenants,
+                        availableIndexNames,
+                        allSeverities,
+                        severitiesFilter,
+                    ]) => {
+                        setTenantId(id);
+                        setIsIngestingMetadataOnly(ingestMetadataOnly);
+                        setIndexName(index);
+                        if (id === -1 && userTenants.length > 0) {
+                            setTenantId(userTenants[0].id);
+                        }
+                        setUserTenants(userTenants);
+                        setIndexNames(availableIndexNames);
+                        setSeverities(allSeverities);
+                        setSelectedSeverities(
+                            convertSeverityFilterToArray(severitiesFilter, allSeverities)
+                        );
                     }
-                    setUserTenants(userTenants);
-                    setIndexNames(availableIndexNames);
-                })
+                )
                 .catch(() => {
                     toastManager.show({
                         id: ToastKeys.ERROR,
@@ -93,11 +122,13 @@ const ConfigurationUserPreferencesStep: FC<{
             setIndexNames([]);
             setUserTenants([]);
             setIsLoading(false);
+            setSeverities([]);
+            setSelectedSeverities([]);
         }
     }, [configurationStep, apiKey]);
 
     const isFormValid = (): boolean => {
-        return tenantId !== undefined;
+        return tenantId !== undefined && selectedSeverities.length > 0;
     };
 
     return (
@@ -129,6 +160,31 @@ const ConfigurationUserPreferencesStep: FC<{
                         })}
                     </Select>
                 </div>
+                <div className="form-item">
+                    <div className="label-tooltip">
+                        <Label>Severity filter</Label>
+                        <Tooltip>
+                            <div>
+                                Select the minimal alert severity to ignore less critical events
+                                associated with this identifier.
+                                <br />
+                                <br />
+                                {'To learn more about severities see '}
+                                <a
+                                    target="_blank"
+                                    href="https://docs.flare.io/understand-event-severity"
+                                >
+                                    Understand Severity Scoring.
+                                </a>
+                            </div>
+                        </Tooltip>
+                    </div>
+                    <SeverityOptions
+                        setSelectedSeverities={setSelectedSeverities}
+                        severities={severities}
+                        selectedSeverities={selectedSeverities}
+                    />
+                </div>
                 <div className="form-item">
                     <div className="label-tooltip">
                         <Label>Basic event ingestion</Label>

packages/react-components/src/components/SeverityOption.css

@@ -0,0 +1,28 @@
+.toggle {
+    position: relative;
+    width: 1rem;
+    height: 1rem;
+    display: inline-block;
+    z-index: 2;
+}
+
+.toggle input {
+    opacity: 0;
+    width: 0;
+    height: 0;
+}
+
+.dot {
+    position: absolute;
+    cursor: pointer;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    height: 1rem;
+    width: 1rem;
+    border-radius: 50%;
+    border-width: 1px;
+    border-style: solid;
+    display: inline-block;
+}