diff --git a/packages/flare/bin/constants.py b/packages/flare/bin/constants.py index e96173e..53d8e3b 100644 --- a/packages/flare/bin/constants.py +++ b/packages/flare/bin/constants.py @@ -15,6 +15,7 @@ class PasswordKeys(Enum): TENANT_ID = "tenant_id" INGEST_METADATA_ONLY = "ingest_metadata_only" SEVERITIES_FILTER = "severities_filter" + SOURCE_TYPES_FILTER = "source_types_filter" class CollectionKeys(Enum): diff --git a/packages/flare/bin/cron_job_ingest_events.py b/packages/flare/bin/cron_job_ingest_events.py index 343b57a..e470a97 100644 --- a/packages/flare/bin/cron_job_ingest_events.py +++ b/packages/flare/bin/cron_job_ingest_events.py @@ -94,6 +94,7 @@ def main( tenant_id = get_tenant_id(storage_passwords=storage_passwords) ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords) severities_filter = get_severities_filter(storage_passwords=storage_passwords) + source_types_filter = get_source_types_filter(storage_passwords=storage_passwords) save_last_fetched(kvstore=kvstore) save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=tenant_id) @@ -105,6 +106,7 @@ def main( tenant_id=tenant_id, ingest_metadata_only=ingest_metadata_only, severities=severities_filter, + source_types=source_types_filter, ): save_last_fetched(kvstore=kvstore) @@ -172,6 +174,18 @@ def get_severities_filter(storage_passwords: StoragePasswords) -> list[str]: return [] +def get_source_types_filter(storage_passwords: StoragePasswords) -> list[str]: + source_types_filter = get_storage_password_value( + storage_passwords=storage_passwords, + password_key=PasswordKeys.SOURCE_TYPES_FILTER.value, + ) + + if source_types_filter: + return source_types_filter.split(",") + + return [] + + def get_next(kvstore: KVStoreCollections, tenant_id: int) -> Optional[str]: return get_collection_value( kvstore=kvstore, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}" @@ -296,6 +310,7 @@ def fetch_feed( tenant_id: int, ingest_metadata_only: bool, severities: list[str], + source_types: list[str], ) -> Iterator[tuple[dict, str]]: try: flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id) @@ -308,6 +323,7 @@ def fetch_feed( start_date=start_date, ingest_metadata_only=ingest_metadata_only, severities=severities, + source_types=source_types, ): yield event_next except Exception as e: diff --git a/packages/flare/bin/flare.py b/packages/flare/bin/flare.py index aed545b..6c2159b 100644 --- a/packages/flare/bin/flare.py +++ b/packages/flare/bin/flare.py @@ -51,11 +51,13 @@ def fetch_feed_events( start_date: Optional[date] = None, ingest_metadata_only: bool, severities: list[str], + source_types: list[str], ) -> Iterator[tuple[dict, str]]: for response in self._fetch_event_feed_metadata( next=next, start_date=start_date, severities=severities, + source_types=source_types, ): event_feed = response.json() self.logger.debug(event_feed) @@ -74,6 +76,7 @@ def _fetch_event_feed_metadata( next: Optional[str] = None, start_date: Optional[date] = None, severities: list[str], + source_types: list[str], ) -> Iterator[requests.Response]: data: Dict[str, Any] = { "from": next if next else None, @@ -89,6 +92,9 @@ def _fetch_event_feed_metadata( if len(severities): data["severity"] = severities + if len(source_types): + data["type"] = source_types + for response in self.flare_client.scroll( method="POST", url="/firework/v4/events/tenant/_search", @@ -118,3 +124,8 @@ def fetch_filters_severity(self) -> requests.Response: return self.flare_client.get( url="/firework/v4/events/filters/severities", ) + + def fetch_filters_source_types(self) -> requests.Response: + return self.flare_client.get( + url="/firework/v4/events/filters/types", + ) diff --git a/packages/flare/bin/flare_external_requests.py b/packages/flare/bin/flare_external_requests.py index 505ef72..8a1e5ed 100644 --- a/packages/flare/bin/flare_external_requests.py +++ b/packages/flare/bin/flare_external_requests.py @@ -58,3 +58,20 @@ def handle_POST(self) -> None: logger.debug(f"FlareFiltersSeverity: {response_json}") self.response.setHeader("Content-Type", "application/json") self.response.write(json.dumps(response_json)) + + +class FlareFiltersSourceTypes(splunk.rest.BaseRestHandler): + def handle_POST(self) -> None: + logger = Logger(class_name=__file__) + payload = self.request["payload"] + params = parse.parse_qs(payload) + + if "apiKey" not in params: + raise Exception("API Key is required") + + flare_api = FlareAPI(api_key=params["apiKey"][0]) + response = flare_api.fetch_filters_source_types() + response_json = response.json() + logger.debug(f"FlareFiltersSourceTypes: {response_json}") + self.response.setHeader("Content-Type", "application/json") + self.response.write(json.dumps(response_json)) diff --git a/packages/flare/src/main/resources/splunk/default/restmap.conf b/packages/flare/src/main/resources/splunk/default/restmap.conf index b161287..35f5fc5 100644 --- a/packages/flare/src/main/resources/splunk/default/restmap.conf +++ b/packages/flare/src/main/resources/splunk/default/restmap.conf @@ -12,3 +12,8 @@ python.version = python3 match=/fetch_filters_severities handler=flare_external_requests.FlareFiltersSeverity python.version = python3 + +[script:flare_external_requests_filters_source_types] +match=/fetch_filters_source_types +handler=flare_external_requests.FlareFiltersSourceTypes +python.version = python3 diff --git a/packages/flare/src/main/resources/splunk/default/web.conf b/packages/flare/src/main/resources/splunk/default/web.conf index 5924200..546b97d 100644 --- a/packages/flare/src/main/resources/splunk/default/web.conf +++ b/packages/flare/src/main/resources/splunk/default/web.conf @@ -9,3 +9,7 @@ methods=POST [expose:flare_external_requests_filters_severities] pattern=fetch_filters_severities methods=POST + +[expose:flare_external_requests_filters_source_types] +pattern=fetch_filters_source_types +methods=POST diff --git a/packages/flare/tests/bin/test_flare_wrapper.py b/packages/flare/tests/bin/test_flare_wrapper.py index 398659b..a40540b 100644 --- a/packages/flare/tests/bin/test_flare_wrapper.py +++ b/packages/flare/tests/bin/test_flare_wrapper.py @@ -45,6 +45,7 @@ def test_flare_full_data_without_metadata( start_date=None, ingest_metadata_only=True, severities=[], + source_types=[], ): assert next_token == expected_return_value["next"] events.append(event) @@ -107,6 +108,7 @@ def test_flare_full_data_with_metadata( start_date=None, ingest_metadata_only=False, severities=[], + source_types=[], ): assert next_token == expected_return_value["next"] events.append(event) @@ -150,6 +152,7 @@ def test_flare_full_data_with_metadata_and_exception( start_date=None, ingest_metadata_only=False, severities=[], + source_types=[], ) ) diff --git a/packages/flare/tests/bin/test_ingest_events.py b/packages/flare/tests/bin/test_ingest_events.py index da0cc40..9433e2d 100644 --- a/packages/flare/tests/bin/test_ingest_events.py +++ b/packages/flare/tests/bin/test_ingest_events.py @@ -229,6 +229,7 @@ def test_fetch_feed_expect_exception() -> None: tenant_id=11111, ingest_metadata_only=False, severities=[], + source_types=[], ): pass @@ -264,6 +265,7 @@ def test_fetch_feed_expect_feed_response( tenant_id=11111, ingest_metadata_only=False, severities=[], + source_types=[], ): assert next_token == next events.append(event) @@ -323,4 +325,5 @@ def test_main_expect_normal_run( tenant_id=111, ingest_metadata_only=False, severities=[], + source_types=[], ) diff --git a/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx b/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx index 7902204..a8a0474 100644 --- a/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx +++ b/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx @@ -1,5 +1,11 @@ import React, { FC, useEffect, useState } from 'react'; -import { ConfigurationStep, Severity, Tenant } from '../models/flare'; +import { + ConfigurationStep, + Severity, + SourceType, + SourceTypeCategory, + Tenant, +} from '../models/flare'; import Button from './Button'; import Label from './Label'; import Select from './Select'; @@ -9,17 +15,22 @@ import { convertSeverityFilterToArray, fetchAvailableIndexNames, fetchCurrentIndexName, + fetchFiltersSourceTypes, fetchFiltersSeverities, fetchIngestMetadataOnly, fetchSeveritiesFilter, fetchTenantId, fetchUserTenants, + fetchSourceTypesFilter, getSeverityFilterValue, + getSourceTypesFilterValue, saveConfiguration, + convertSourceTypeFilterToArray, } from '../utils/setupConfiguration'; import './ConfigurationGlobalStep.css'; import './ConfigurationUserPreferencesStep.css'; import SeverityOptions from './SeverityOptions'; +import SourceTypeCategoryOptions from './SourceTypeCategoryOptions'; import Switch from './Switch'; import { ToastKeys, toastManager } from './ToastManager'; import Tooltip from './Tooltip'; @@ -35,6 +46,8 @@ const ConfigurationUserPreferencesStep: FC<{ const [tenants, setUserTenants] = useState([]); const [selectedSeverities, setSelectedSeverities] = useState([]); const [severities, setSeverities] = useState([]); + const [sourceTypeCategories, setSourceTypeCategories] = useState([]); + const [selectedSourceTypes, setSelectedSourceTypes] = useState([]); const [indexName, setIndexName] = useState(''); const [indexNames, setIndexNames] = useState([]); const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false); @@ -53,7 +66,8 @@ const ConfigurationUserPreferencesStep: FC<{ Number(tenantId), indexName, isIngestingMetadataOnly, - getSeverityFilterValue(selectedSeverities, severities) + getSeverityFilterValue(selectedSeverities, severities), + getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories) ) .then(() => { setIsLoading(false); @@ -84,6 +98,8 @@ const ConfigurationUserPreferencesStep: FC<{ fetchAvailableIndexNames(), fetchFiltersSeverities(apiKey), fetchSeveritiesFilter(), + fetchFiltersSourceTypes(apiKey), + fetchSourceTypesFilter(), ]) .then( ([ @@ -94,6 +110,8 @@ const ConfigurationUserPreferencesStep: FC<{ availableIndexNames, availableSeverities, severitiesFilter, + availableSourceTypeCategories, + sourceTypeFilter, ]) => { setTenantId(id); setIsIngestingMetadataOnly(ingestMetadataOnly); @@ -107,6 +125,13 @@ const ConfigurationUserPreferencesStep: FC<{ setSelectedSeverities( convertSeverityFilterToArray(severitiesFilter, availableSeverities) ); + setSourceTypeCategories(availableSourceTypeCategories); + setSelectedSourceTypes( + convertSourceTypeFilterToArray( + sourceTypeFilter, + availableSourceTypeCategories + ) + ); } ) .catch(() => { @@ -124,11 +149,17 @@ const ConfigurationUserPreferencesStep: FC<{ setIsLoading(false); setSeverities([]); setSelectedSeverities([]); + setSourceTypeCategories([]); + setSelectedSourceTypes([]); } }, [configurationStep, apiKey]); const isFormValid = (): boolean => { - return tenantId !== undefined && selectedSeverities.length > 0; + return ( + tenantId !== undefined && + selectedSeverities.length > 0 && + selectedSourceTypes.length > 0 + ); }; return ( @@ -185,6 +216,27 @@ const ConfigurationUserPreferencesStep: FC<{ selectedSeverities={selectedSeverities} /> +
+
+ + +
+ {'For more details on Identifier Categories, please visit our '} + + Documentation. + +
+ +
+ +
diff --git a/packages/react-components/src/components/SourceTypeCategoryOption.css b/packages/react-components/src/components/SourceTypeCategoryOption.css new file mode 100644 index 0000000..f6b994f --- /dev/null +++ b/packages/react-components/src/components/SourceTypeCategoryOption.css @@ -0,0 +1,56 @@ +.source-types-category-container { + display: flex; + flex-direction: column; +} + +.source-types-children-container { + margin-left: 2rem; + margin-top: 0.25rem; + display: flex; + flex-direction: column; + align-items: start; + gap: 0.25rem; +} + +.source-types-children-container[hidden] { + display: none; +} + +.source-types-category-header { + width: 14rem; + display: flex; + flex-direction: row; +} + +.source-types-category-count { + user-select: none; + color: var(--secondary-text-color); + flex: 1; + text-align: end; +} + +.source-types-category { + width: 1rem; + height: 1rem; + margin-left: 0.5rem; + cursor: pointer; + align-self: center; +} + +.source-types-category > span { + display: inline-table; + width: 0.5rem; + height: 0.5rem; + border: solid white; + border-width: 0 0 0.125rem 0.125rem; +} + +.source-types-category-expand > span { + transform: rotate(-45deg); + margin-bottom: 0.5rem; +} + +.source-types-category-collapse > span { + transform: rotate(135deg); + margin-top: 0.25rem; +} diff --git a/packages/react-components/src/components/SourceTypeCategoryOption.tsx b/packages/react-components/src/components/SourceTypeCategoryOption.tsx new file mode 100644 index 0000000..cdfd0ae --- /dev/null +++ b/packages/react-components/src/components/SourceTypeCategoryOption.tsx @@ -0,0 +1,76 @@ +import React, { FC, useState } from 'react'; + +import { SourceType, SourceTypeCategory } from '../models/flare'; +import './SourceTypeCategoryOption.css'; +import SourceTypeOption from './SourceTypeOption'; + +const SourceTypeCategoryOption: FC<{ + isChecked?: boolean; + sourceTypeCategory: SourceTypeCategory; + isSourceTypeChecked: (sourceType: SourceType) => boolean; + onCategoryCheckChange: (isChecked: boolean) => void; + onSourceTypeCheckChange: (sourceType: SourceType, isChecked: boolean) => void; +}> = ({ + isChecked = false, + sourceTypeCategory, + isSourceTypeChecked, + onCategoryCheckChange, + onSourceTypeCheckChange, +}) => { + const [isExpanded, setExpanded] = useState(true); + + const getSelectedCategoryCount = (): number => { + return sourceTypeCategory.types.filter((sourceType) => isSourceTypeChecked(sourceType)) + .length; + }; + + const selectedCategoryCount = getSelectedCategoryCount(); + + return ( +
+
+ 0 && !isChecked} + onCheckChange={(checked): void => onCategoryCheckChange(checked)} + /> + + + +
+ +
+ ); +}; + +export default SourceTypeCategoryOption; diff --git a/packages/react-components/src/components/SourceTypeCategoryOptions.css b/packages/react-components/src/components/SourceTypeCategoryOptions.css new file mode 100644 index 0000000..3a2f3d7 --- /dev/null +++ b/packages/react-components/src/components/SourceTypeCategoryOptions.css @@ -0,0 +1,8 @@ +#source-types-categories-container { + display: flex; + flex-direction: column; + margin-top: 0.5rem; + gap: 1rem; + align-items: start; + text-align: start; +} diff --git a/packages/react-components/src/components/SourceTypeCategoryOptions.tsx b/packages/react-components/src/components/SourceTypeCategoryOptions.tsx new file mode 100644 index 0000000..d9ece35 --- /dev/null +++ b/packages/react-components/src/components/SourceTypeCategoryOptions.tsx @@ -0,0 +1,79 @@ +import React, { FC } from 'react'; + +import { SourceType, SourceTypeCategory } from '../models/flare'; +import SourceTypeCategoryOption from './SourceTypeCategoryOption'; +import './SourceTypeCategoryOptions.css'; + +const SourceTypeCategoryOptions: FC<{ + sourceTypeCategories: SourceTypeCategory[]; + selectedSourceTypes: SourceType[]; + setSelectedSourceTypes: (selectedSourceTypes: SourceType[]) => void; +}> = ({ sourceTypeCategories, selectedSourceTypes, setSelectedSourceTypes }) => { + const isSourceTypeChecked = (sourceType: SourceType): boolean => { + return ( + selectedSourceTypes.findIndex( + (selectedSourceType) => selectedSourceType.value === sourceType.value + ) >= 0 + ); + }; + + const isSourceTypeCategoryChecked = (sourceTypeCategory: SourceTypeCategory): boolean => { + return ( + sourceTypeCategory.types.filter((sourceType) => { + return isSourceTypeChecked(sourceType); + }).length === sourceTypeCategory.types.length + ); + }; + + const handleOnSourceTypeChange = (sourceType: SourceType, isChecked: boolean): void => { + if (isChecked) { + const newSourceTypes = new Array(...selectedSourceTypes); + newSourceTypes.push(sourceType); + setSelectedSourceTypes(newSourceTypes); + } else { + const newSourceTypes = selectedSourceTypes.filter( + (selectedSourceType) => selectedSourceType.value !== sourceType.value + ); + setSelectedSourceTypes(newSourceTypes); + } + }; + + const handleOnSourceTypeCategoryChange = ( + sourceTypeCategory: SourceTypeCategory, + isChecked: boolean + ): void => { + if (isChecked) { + const newSourceTypes = new Set(selectedSourceTypes); + sourceTypeCategory.types.forEach((sourceType) => newSourceTypes.add(sourceType)); + setSelectedSourceTypes(new Array(...newSourceTypes)); + } else { + const newSourceTypes = selectedSourceTypes.filter( + (selectedSourceType) => sourceTypeCategory.types.indexOf(selectedSourceType) === -1 + ); + setSelectedSourceTypes(newSourceTypes); + } + }; + + return ( +
+ {sourceTypeCategories.map((sourceTypeCategory) => { + return ( + + handleOnSourceTypeCategoryChange(sourceTypeCategory, checked) + } + onSourceTypeCheckChange={(sourceType, checked): void => + handleOnSourceTypeChange(sourceType, checked) + } + /> + ); + })} +
+ ); +}; + +export default SourceTypeCategoryOptions; diff --git a/packages/react-components/src/components/SourceTypeOption.css b/packages/react-components/src/components/SourceTypeOption.css new file mode 100644 index 0000000..d6969ce --- /dev/null +++ b/packages/react-components/src/components/SourceTypeOption.css @@ -0,0 +1,82 @@ +.source-type-container { + user-select: none; + display: flex; + flex-direction: row; + gap: 0.5rem; + cursor: pointer; + align-items: center; +} + +.source-type-option-input { + width: 0; + height: 0; + opacity: 0; +} + +.source-type-checkbox { + width: 1rem; + height: 1rem; + background-color: var(--switch-disabled-bg-color); + border: 1px solid var(--button-bg-color); + border-radius: 0.125rem; + display: flex; + justify-content: center; + align-items: center; + position: relative; +} + +.source-type-container:hover .source-type-checkbox { + background-color: var(--switch-disabled-hover-bg-color); +} + +.source-type-checkbox-partial { + width: 1rem; + height: 1rem; + background-color: var(--button-bg-color); + border: 1px solid var(--button-bg-color); + border-radius: 0.125rem; + display: flex; + justify-content: center; + align-items: center; + position: relative; +} + +.source-type-container:hover .source-type-checkbox-partial { + background-color: var(--button-bg-hover-color); +} + +.source-type-checkbox-partial::after { + content: ''; + width: 0.5rem; + height: 0; + border: solid white; + border-width: 0 0 0.125rem 0.125rem; + position: absolute; +} + +input:checked + .source-type-checkbox { + width: 1rem; + height: 1rem; + background-color: var(--button-bg-color); + border: 1px solid var(--button-bg-color); + border-radius: 0.125rem; + display: flex; + justify-content: center; + align-items: center; + position: relative; +} + +.source-type-container:hover:has(> input:checked) .source-type-checkbox { + background-color: var(--button-bg-hover-color); +} + +input:checked + .source-type-checkbox::after { + content: ''; + width: 0.5rem; + height: 0.25rem; + border: solid white; + border-width: 0 0 0.125rem 0.125rem; + transform: rotate(-45deg); + position: absolute; + margin-bottom: 0.125rem; +} diff --git a/packages/react-components/src/components/SourceTypeOption.tsx b/packages/react-components/src/components/SourceTypeOption.tsx new file mode 100644 index 0000000..39ea2a4 --- /dev/null +++ b/packages/react-components/src/components/SourceTypeOption.tsx @@ -0,0 +1,31 @@ +import React, { FC } from 'react'; + +import { SourceType } from '../models/flare'; +import './SourceTypeOption.css'; + +const SourceTypeOption: FC<{ + isChecked?: boolean; + isPartiallyChecked?: boolean; + sourceType: SourceType; + onCheckChange: (isChecked: boolean) => void; +}> = ({ isChecked = false, isPartiallyChecked = false, sourceType, onCheckChange }) => { + return ( +