Source types filter

- Fetch available source types via the rest API
- Added picker in the user preference configuration step
- Unit tests
- Logic filtering is:
	- Minimum one source type must be selected
	- If all of the source types are selected, no filter is applied in case we add new source type categories/sub types in the future
	- If all of the subtypes of a category is selected, the name of the category will be passed in the filter instead of each children in case the sub types are changed in the future
	- Otherwise, pass the source types selected as a filter when fetching the events

packages/flare/bin/constants.py

@@ -15,6 +15,7 @@ class PasswordKeys(Enum):
     TENANT_ID = "tenant_id"
     INGEST_METADATA_ONLY = "ingest_metadata_only"
     SEVERITIES_FILTER = "severities_filter"
+    SOURCE_TYPES_FILTER = "source_types_filter"
 
 
 class CollectionKeys(Enum):

packages/flare/bin/cron_job_ingest_events.py

@@ -94,6 +94,7 @@ def main(
     tenant_id = get_tenant_id(storage_passwords=storage_passwords)
     ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords)
     severities_filter = get_severities_filter(storage_passwords=storage_passwords)
+    source_types_filter = get_source_types_filter(storage_passwords=storage_passwords)
 
     save_last_fetched(kvstore=kvstore)
     save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=tenant_id)
@@ -105,6 +106,7 @@ def main(
         tenant_id=tenant_id,
         ingest_metadata_only=ingest_metadata_only,
         severities=severities_filter,
+        source_types=source_types_filter,
     ):
         save_last_fetched(kvstore=kvstore)
 
@@ -172,6 +174,18 @@ def get_severities_filter(storage_passwords: StoragePasswords) -> list[str]:
     return []
 
 
+def get_source_types_filter(storage_passwords: StoragePasswords) -> list[str]:
+    source_types_filter = get_storage_password_value(
+        storage_passwords=storage_passwords,
+        password_key=PasswordKeys.SOURCE_TYPES_FILTER.value,
+    )
+
+    if source_types_filter:
+        return source_types_filter.split(",")
+
+    return []
+
+
 def get_next(kvstore: KVStoreCollections, tenant_id: int) -> Optional[str]:
     return get_collection_value(
         kvstore=kvstore, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}"
@@ -296,6 +310,7 @@ def fetch_feed(
     tenant_id: int,
     ingest_metadata_only: bool,
     severities: list[str],
+    source_types: list[str],
 ) -> Iterator[tuple[dict, str]]:
     try:
         flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id)
@@ -308,6 +323,7 @@ def fetch_feed(
             start_date=start_date,
             ingest_metadata_only=ingest_metadata_only,
             severities=severities,
+            source_types=source_types,
         ):
             yield event_next
     except Exception as e:

packages/flare/bin/flare.py

@@ -51,11 +51,13 @@ def fetch_feed_events(
         start_date: Optional[date] = None,
         ingest_metadata_only: bool,
         severities: list[str],
+        source_types: list[str],
     ) -> Iterator[tuple[dict, str]]:
         for response in self._fetch_event_feed_metadata(
             next=next,
             start_date=start_date,
             severities=severities,
+            source_types=source_types,
         ):
             event_feed = response.json()
             self.logger.debug(event_feed)
@@ -74,6 +76,7 @@ def _fetch_event_feed_metadata(
         next: Optional[str] = None,
         start_date: Optional[date] = None,
         severities: list[str],
+        source_types: list[str],
     ) -> Iterator[requests.Response]:
         data: Dict[str, Any] = {
             "from": next if next else None,
@@ -89,6 +92,9 @@ def _fetch_event_feed_metadata(
         if len(severities):
             data["severity"] = severities
 
+        if len(source_types):
+            data["type"] = source_types
+
         for response in self.flare_client.scroll(
             method="POST",
             url="/firework/v4/events/tenant/_search",
@@ -118,3 +124,8 @@ def fetch_filters_severity(self) -> requests.Response:
         return self.flare_client.get(
             url="/firework/v4/events/filters/severities",
         )
+
+    def fetch_filters_source_types(self) -> requests.Response:
+        return self.flare_client.get(
+            url="/firework/v4/events/filters/types",
+        )

packages/flare/bin/flare_external_requests.py

@@ -58,3 +58,20 @@ def handle_POST(self) -> None:
         logger.debug(f"FlareFiltersSeverity: {response_json}")
         self.response.setHeader("Content-Type", "application/json")
         self.response.write(json.dumps(response_json))
+
+
+class FlareFiltersSourceTypes(splunk.rest.BaseRestHandler):
+    def handle_POST(self) -> None:
+        logger = Logger(class_name=__file__)
+        payload = self.request["payload"]
+        params = parse.parse_qs(payload)
+
+        if "apiKey" not in params:
+            raise Exception("API Key is required")
+
+        flare_api = FlareAPI(api_key=params["apiKey"][0])
+        response = flare_api.fetch_filters_source_types()
+        response_json = response.json()
+        logger.debug(f"FlareFiltersSourceTypes: {response_json}")
+        self.response.setHeader("Content-Type", "application/json")
+        self.response.write(json.dumps(response_json))

packages/flare/src/main/resources/splunk/default/restmap.conf

@@ -12,3 +12,8 @@ python.version = python3
 match=/fetch_filters_severities
 handler=flare_external_requests.FlareFiltersSeverity
 python.version = python3
+
+[script:flare_external_requests_filters_source_types]
+match=/fetch_filters_source_types
+handler=flare_external_requests.FlareFiltersSourceTypes
+python.version = python3

packages/flare/src/main/resources/splunk/default/web.conf

@@ -9,3 +9,7 @@ methods=POST
 [expose:flare_external_requests_filters_severities]
 pattern=fetch_filters_severities
 methods=POST
+
+[expose:flare_external_requests_filters_source_types]
+pattern=fetch_filters_source_types
+methods=POST

packages/flare/tests/bin/test_flare_wrapper.py

@@ -45,6 +45,7 @@ def test_flare_full_data_without_metadata(
         start_date=None,
         ingest_metadata_only=True,
         severities=[],
+        source_types=[],
     ):
         assert next_token == expected_return_value["next"]
         events.append(event)
@@ -107,6 +108,7 @@ def test_flare_full_data_with_metadata(
         start_date=None,
         ingest_metadata_only=False,
         severities=[],
+        source_types=[],
     ):
         assert next_token == expected_return_value["next"]
         events.append(event)
@@ -150,6 +152,7 @@ def test_flare_full_data_with_metadata_and_exception(
                 start_date=None,
                 ingest_metadata_only=False,
                 severities=[],
+                source_types=[],
             )
         )
 

packages/flare/tests/bin/test_ingest_events.py

@@ -229,6 +229,7 @@ def test_fetch_feed_expect_exception() -> None:
         tenant_id=11111,
         ingest_metadata_only=False,
         severities=[],
+        source_types=[],
     ):
         pass
 
@@ -264,6 +265,7 @@ def test_fetch_feed_expect_feed_response(
         tenant_id=11111,
         ingest_metadata_only=False,
         severities=[],
+        source_types=[],
     ):
         assert next_token == next
         events.append(event)
@@ -323,4 +325,5 @@ def test_main_expect_normal_run(
         tenant_id=111,
         ingest_metadata_only=False,
         severities=[],
+        source_types=[],
     )

packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx

@@ -1,5 +1,11 @@
 import React, { FC, useEffect, useState } from 'react';
-import { ConfigurationStep, Severity, Tenant } from '../models/flare';
+import {
+    ConfigurationStep,
+    Severity,
+    SourceType,
+    SourceTypeCategory,
+    Tenant,
+} from '../models/flare';
 import Button from './Button';
 import Label from './Label';
 import Select from './Select';
@@ -9,17 +15,22 @@ import {
     convertSeverityFilterToArray,
     fetchAvailableIndexNames,
     fetchCurrentIndexName,
+    fetchFiltersSourceTypes,
     fetchFiltersSeverities,
     fetchIngestMetadataOnly,
     fetchSeveritiesFilter,
     fetchTenantId,
     fetchUserTenants,
+    fetchSourceTypesFilter,
     getSeverityFilterValue,
+    getSourceTypesFilterValue,
     saveConfiguration,
+    convertSourceTypeFilterToArray,
 } from '../utils/setupConfiguration';
 import './ConfigurationGlobalStep.css';
 import './ConfigurationUserPreferencesStep.css';
 import SeverityOptions from './SeverityOptions';
+import SourceTypeCategoryOptions from './SourceTypeCategoryOptions';
 import Switch from './Switch';
 import { ToastKeys, toastManager } from './ToastManager';
 import Tooltip from './Tooltip';
@@ -35,6 +46,8 @@ const ConfigurationUserPreferencesStep: FC<{
     const [tenants, setUserTenants] = useState<Tenant[]>([]);
     const [selectedSeverities, setSelectedSeverities] = useState<Severity[]>([]);
     const [severities, setSeverities] = useState<Severity[]>([]);
+    const [sourceTypeCategories, setSourceTypeCategories] = useState<SourceTypeCategory[]>([]);
+    const [selectedSourceTypes, setSelectedSourceTypes] = useState<SourceType[]>([]);
     const [indexName, setIndexName] = useState('');
     const [indexNames, setIndexNames] = useState<string[]>([]);
     const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false);
@@ -53,7 +66,8 @@ const ConfigurationUserPreferencesStep: FC<{
             Number(tenantId),
             indexName,
             isIngestingMetadataOnly,
-            getSeverityFilterValue(selectedSeverities, severities)
+            getSeverityFilterValue(selectedSeverities, severities),
+            getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories)
         )
             .then(() => {
                 setIsLoading(false);
@@ -84,6 +98,8 @@ const ConfigurationUserPreferencesStep: FC<{
                 fetchAvailableIndexNames(),
                 fetchFiltersSeverities(apiKey),
                 fetchSeveritiesFilter(),
+                fetchFiltersSourceTypes(apiKey),
+                fetchSourceTypesFilter(),
             ])
                 .then(
                     ([
@@ -94,6 +110,8 @@ const ConfigurationUserPreferencesStep: FC<{
                         availableIndexNames,
                         availableSeverities,
                         severitiesFilter,
+                        availableSourceTypeCategories,
+                        sourceTypeFilter,
                     ]) => {
                         setTenantId(id);
                         setIsIngestingMetadataOnly(ingestMetadataOnly);
@@ -107,6 +125,13 @@ const ConfigurationUserPreferencesStep: FC<{
                         setSelectedSeverities(
                             convertSeverityFilterToArray(severitiesFilter, availableSeverities)
                         );
+                        setSourceTypeCategories(availableSourceTypeCategories);
+                        setSelectedSourceTypes(
+                            convertSourceTypeFilterToArray(
+                                sourceTypeFilter,
+                                availableSourceTypeCategories
+                            )
+                        );
                     }
                 )
                 .catch(() => {
@@ -124,11 +149,17 @@ const ConfigurationUserPreferencesStep: FC<{
             setIsLoading(false);
             setSeverities([]);
             setSelectedSeverities([]);
+            setSourceTypeCategories([]);
+            setSelectedSourceTypes([]);
         }
     }, [configurationStep, apiKey]);
 
     const isFormValid = (): boolean => {
-        return tenantId !== undefined && selectedSeverities.length > 0;
+        return (
+            tenantId !== undefined &&
+            selectedSeverities.length > 0 &&
+            selectedSourceTypes.length > 0
+        );
     };
 
     return (
@@ -185,6 +216,27 @@ const ConfigurationUserPreferencesStep: FC<{
                         selectedSeverities={selectedSeverities}
                     />
                 </div>
+                <div className="form-item">
+                    <div className="label-tooltip">
+                        <Label>Categories filter</Label>
+                        <Tooltip>
+                            <div>
+                                {'For more details on Identifier Categories, please visit our '}
+                                <a
+                                    target="_blank"
+                                    href="https://docs.flare.io/configure-identifiers"
+                                >
+                                    Documentation.
+                                </a>
+                            </div>
+                        </Tooltip>
+                    </div>
+                    <SourceTypeCategoryOptions
+                        setSelectedSourceTypes={setSelectedSourceTypes}
+                        sourceTypeCategories={sourceTypeCategories}
+                        selectedSourceTypes={selectedSourceTypes}
+                    />
+                </div>
                 <div className="form-item">
                     <div className="label-tooltip">
                         <Label>Basic event ingestion</Label>

packages/react-components/src/components/SourceTypeCategoryOption.css

@@ -0,0 +1,56 @@
+.source-types-category-container {
+    display: flex;
+    flex-direction: column;
+}
+
+.source-types-children-container {
+    margin-left: 2rem;
+    margin-top: 0.25rem;
+    display: flex;
+    flex-direction: column;
+    align-items: start;
+    gap: 0.25rem;
+}
+
+.source-types-children-container[hidden] {
+    display: none;
+}
+
+.source-types-category-header {
+    width: 14rem;
+    display: flex;
+    flex-direction: row;
+}
+
+.source-types-category-count {
+    user-select: none;
+    color: var(--secondary-text-color);
+    flex: 1;
+    text-align: end;
+}
+
+.source-types-category {
+    width: 1rem;
+    height: 1rem;
+    margin-left: 0.5rem;
+    cursor: pointer;
+    align-self: center;
+}
+
+.source-types-category > span {
+    display: inline-table;
+    width: 0.5rem;
+    height: 0.5rem;
+    border: solid white;
+    border-width: 0 0 0.125rem 0.125rem;
+}
+
+.source-types-category-expand > span {
+    transform: rotate(-45deg);
+    margin-bottom: 0.5rem;
+}
+
+.source-types-category-collapse > span {
+    transform: rotate(135deg);
+    margin-top: 0.25rem;
+}