diff --git a/packages/flare/bin/constants.py b/packages/flare/bin/constants.py index 6374a9b..a06b30f 100644 --- a/packages/flare/bin/constants.py +++ b/packages/flare/bin/constants.py @@ -8,12 +8,15 @@ REALM = APP_NAME + "_realm" KV_COLLECTION_NAME = "event_ingestion_collection" CRON_JOB_THRESHOLD_SINCE_LAST_FETCH = timedelta(minutes=10) +NO_FILTER_VALUE = "none" class PasswordKeys(Enum): API_KEY = "api_key" TENANT_ID = "tenant_id" INGEST_METADATA_ONLY = "ingest_metadata_only" + SEVERITIES_FILTER = "severities_filter" + SOURCE_TYPES_FILTER = "source_types_filter" class CollectionKeys(Enum): diff --git a/packages/flare/bin/cron_job_ingest_events.py b/packages/flare/bin/cron_job_ingest_events.py index e50ceb6..1fce9fa 100644 --- a/packages/flare/bin/cron_job_ingest_events.py +++ b/packages/flare/bin/cron_job_ingest_events.py @@ -16,6 +16,7 @@ from constants import CRON_JOB_THRESHOLD_SINCE_LAST_FETCH from constants import HOST from constants import KV_COLLECTION_NAME +from constants import NO_FILTER_VALUE from constants import SPLUNK_PORT from constants import CollectionKeys from constants import PasswordKeys @@ -40,6 +41,8 @@ def main(logger: Logger, app: client.Application) -> None: api_key = get_api_key(app=app) tenant_id = get_tenant_id(app=app) ingest_metadata_only = get_ingest_metadata_only(app=app) + severities_filter = get_severities_filter(app=app) + source_types_filter = get_source_types_filter(app=app) save_last_fetched(app=app) save_last_ingested_tenant_id(app=app, tenant_id=tenant_id) @@ -50,6 +53,8 @@ def main(logger: Logger, app: client.Application) -> None: api_key=api_key, tenant_id=tenant_id, ingest_metadata_only=ingest_metadata_only, + severities=severities_filter, + source_types=source_types_filter, ): save_last_fetched(app=app) @@ -104,6 +109,28 @@ def get_ingest_metadata_only(app: client.Application) -> bool: ) +def get_severities_filter(app: client.Application) -> list[str]: + severities_filter = get_storage_password_value( + app=app, password_key=PasswordKeys.SEVERITIES_FILTER.value + ) + + if severities_filter and severities_filter != NO_FILTER_VALUE: + return severities_filter.split(",") + + return [] + + +def get_source_types_filter(app: client.Application) -> list[str]: + source_types_filter = get_storage_password_value( + app=app, password_key=PasswordKeys.SOURCE_TYPES_FILTER.value + ) + + if source_types_filter and source_types_filter != NO_FILTER_VALUE: + return source_types_filter.split(",") + + return [] + + def get_next(app: client.Application, tenant_id: int) -> Optional[str]: return get_collection_value( app=app, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}" @@ -222,6 +249,8 @@ def fetch_feed( api_key: str, tenant_id: int, ingest_metadata_only: bool, + severities: list[str], + source_types: list[str], ) -> Iterator[tuple[dict, str]]: try: flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id) @@ -230,7 +259,11 @@ def fetch_feed( start_date = get_start_date(app=app) logger.info(f"Fetching {tenant_id=}, {next=}, {start_date=}") for event_next in flare_api.fetch_feed_events( - next=next, start_date=start_date, ingest_metadata_only=ingest_metadata_only + next=next, + start_date=start_date, + ingest_metadata_only=ingest_metadata_only, + severities=severities, + source_types=source_types, ): yield event_next except Exception as e: diff --git a/packages/flare/bin/flare.py b/packages/flare/bin/flare.py index e1177e9..e99af3f 100644 --- a/packages/flare/bin/flare.py +++ b/packages/flare/bin/flare.py @@ -50,10 +50,14 @@ def fetch_feed_events( next: Optional[str] = None, start_date: Optional[date] = None, ingest_metadata_only: bool, + severities: list[str], + source_types: list[str], ) -> Iterator[tuple[dict, str]]: for response in self._fetch_event_feed_metadata( next=next, start_date=start_date, + severities=severities, + source_types=source_types, ): event_feed = response.json() self.logger.debug(event_feed) @@ -71,6 +75,8 @@ def _fetch_event_feed_metadata( *, next: Optional[str] = None, start_date: Optional[date] = None, + severities: list[str], + source_types: list[str], ) -> Iterator[requests.Response]: data: Dict[str, Any] = { "from": next if next else None, @@ -79,7 +85,9 @@ def _fetch_event_feed_metadata( "gte": start_date.isoformat() if start_date else date.today().isoformat() - } + }, + "severity": severities if len(severities) > 0 else None, + "types": source_types if len(source_types) > 0 else None, }, } @@ -98,7 +106,22 @@ def _fetch_full_event_from_uid(self, *, uid: str) -> dict: self.logger.debug(event) return event + def fetch_api_key_validation(self) -> requests.Response: + return self.flare_client.get( + url="/tokens/test", + ) + def fetch_tenants(self) -> requests.Response: return self.flare_client.get( url="/firework/v2/me/tenants", ) + + def fetch_filters_severity(self) -> requests.Response: + return self.flare_client.get( + url="/firework/v4/events/filters/severities", + ) + + def fetch_filters_source_types(self) -> requests.Response: + return self.flare_client.get( + url="/firework/v4/events/filters/types", + ) diff --git a/packages/flare/bin/flare_external_requests.py b/packages/flare/bin/flare_external_requests.py index 4eb4eed..65a75f4 100644 --- a/packages/flare/bin/flare_external_requests.py +++ b/packages/flare/bin/flare_external_requests.py @@ -12,6 +12,20 @@ from logger import Logger +class FlareValidateApiKey(splunk.rest.BaseRestHandler): + def handle_POST(self) -> None: + payload = self.request["payload"] + params = parse.parse_qs(payload) + + if "apiKey" in params: + flare_api = FlareAPI(api_key=params["apiKey"][0]) + flare_api.fetch_api_key_validation() + self.response.setHeader("Content-Type", "application/json") + self.response.write(json.dumps({})) + else: + raise Exception("API Key is required") + + class FlareUserTenants(splunk.rest.BaseRestHandler): def handle_POST(self) -> None: logger = Logger(class_name=__file__) @@ -20,10 +34,44 @@ def handle_POST(self) -> None: if "apiKey" in params: flare_api = FlareAPI(api_key=params["apiKey"][0]) - user_tenants_response = flare_api.fetch_tenants() - tenants_response = user_tenants_response.json() - logger.debug(tenants_response) + response = flare_api.fetch_tenants() + response_json = response.json() + logger.debug(f"FlareUserTenants: {response_json}") + self.response.setHeader("Content-Type", "application/json") + self.response.write(json.dumps(response_json)) + else: + raise Exception("API Key is required") + + +class FlareFiltersSeverity(splunk.rest.BaseRestHandler): + def handle_POST(self) -> None: + logger = Logger(class_name=__file__) + payload = self.request["payload"] + params = parse.parse_qs(payload) + + if "apiKey" in params: + flare_api = FlareAPI(api_key=params["apiKey"][0]) + response = flare_api.fetch_filters_severity() + response_json = response.json() + logger.debug(f"FlareFiltersSeverity: {response_json}") + self.response.setHeader("Content-Type", "application/json") + self.response.write(json.dumps(response_json)) + else: + raise Exception("API Key is required") + + +class FlareFiltersSourceTypes(splunk.rest.BaseRestHandler): + def handle_POST(self) -> None: + logger = Logger(class_name=__file__) + payload = self.request["payload"] + params = parse.parse_qs(payload) + + if "apiKey" in params: + flare_api = FlareAPI(api_key=params["apiKey"][0]) + response = flare_api.fetch_filters_source_types() + response_json = response.json() + logger.debug(f"FlareFiltersSourceTypes: {response_json}") self.response.setHeader("Content-Type", "application/json") - self.response.write(json.dumps(tenants_response)) + self.response.write(json.dumps(response_json)) else: raise Exception("API Key is required") diff --git a/packages/flare/src/main/resources/splunk/default/restmap.conf b/packages/flare/src/main/resources/splunk/default/restmap.conf index e5b7847..35f5fc5 100644 --- a/packages/flare/src/main/resources/splunk/default/restmap.conf +++ b/packages/flare/src/main/resources/splunk/default/restmap.conf @@ -1,4 +1,19 @@ +[script:flare_external_requests_api_key_validation] +match=/fetch_api_key_validation +handler=flare_external_requests.FlareValidateApiKey +python.version = python3 + [script:flare_external_requests_user_tenants] match=/fetch_user_tenants handler=flare_external_requests.FlareUserTenants python.version = python3 + +[script:flare_external_requests_filters_severities] +match=/fetch_filters_severities +handler=flare_external_requests.FlareFiltersSeverity +python.version = python3 + +[script:flare_external_requests_filters_source_types] +match=/fetch_filters_source_types +handler=flare_external_requests.FlareFiltersSourceTypes +python.version = python3 diff --git a/packages/flare/src/main/resources/splunk/default/web.conf b/packages/flare/src/main/resources/splunk/default/web.conf index e310a5d..546b97d 100644 --- a/packages/flare/src/main/resources/splunk/default/web.conf +++ b/packages/flare/src/main/resources/splunk/default/web.conf @@ -1,3 +1,15 @@ +[expose:flare_external_requests_api_key_validation] +pattern=fetch_api_key_validation +methods=POST + [expose:flare_external_requests_user_tenants] pattern=fetch_user_tenants methods=POST + +[expose:flare_external_requests_filters_severities] +pattern=fetch_filters_severities +methods=POST + +[expose:flare_external_requests_filters_source_types] +pattern=fetch_filters_source_types +methods=POST diff --git a/packages/react-components/src/ConfigurationScreen.css b/packages/react-components/src/ConfigurationScreen.css index 40d7782..20b846b 100644 --- a/packages/react-components/src/ConfigurationScreen.css +++ b/packages/react-components/src/ConfigurationScreen.css @@ -17,6 +17,7 @@ width: 800px; gap: 1.5rem; align-self: center; + padding-bottom: 2rem; } .content-step { diff --git a/packages/react-components/src/ConfigurationScreen.tsx b/packages/react-components/src/ConfigurationScreen.tsx index 35f8996..84502e6 100644 --- a/packages/react-components/src/ConfigurationScreen.tsx +++ b/packages/react-components/src/ConfigurationScreen.tsx @@ -1,156 +1,40 @@ -import React, { useEffect, useState, FC } from 'react'; -import { - appName, - createFlareIndex, - redirectToHomepage, - fetchApiKey, - fetchAvailableIndexNames, - fetchCurrentIndexName, - fetchIngestMetadataOnly, - fetchTenantId, - fetchUserTenants, - saveConfiguration, -} from './utils/setupConfiguration'; -import { ConfigurationSteps, Tenant } from './models/flare'; -import './global.css'; +import React, { FC, useEffect, useState } from 'react'; import './ConfigurationScreen.css'; +import ConfigurationCompletedStep from './components/ConfigurationCompletedStep'; +import ConfigurationInitialStep from './components/ConfigurationInitialStep'; +import ConfigurationUserPreferencesStep from './components/ConfigurationUserPreferencesStep'; import LoadingBar from './components/LoadingBar'; +import { toastManager } from './components/ToastManager'; import DoneIcon from './components/icons/DoneIcon'; import ExternalLinkIcon from './components/icons/ExternalLinkIcon'; -import { toastManager } from './components/ToastManager'; import ToolIcon from './components/icons/ToolIcon'; -import ConfigurationInitialStep from './components/ConfigurationInitialStep'; -import ConfigurationUserPreferencesStep from './components/ConfigurationUserPreferencesStep'; -import ConfigurationCompletedStep from './components/ConfigurationCompletedStep'; - -const TOAST_API_KEY_ERROR = 'api_key_error'; -const TOAST_TENANT_SUCCESS = 'tenant_success'; +import './global.css'; +import { ConfigurationStep } from './models/flare'; +import { createFlareIndex, fetchApiKey, redirectToHomepage } from './utils/setupConfiguration'; const ConfigurationScreen: FC<{ theme: string }> = ({ theme }) => { + const [configurationStep, setConfigurationStep] = useState(ConfigurationStep.Initial); const [apiKey, setApiKey] = useState(''); - const [tenantId, setTenantId] = useState(-1); - const [indexName, setIndexName] = useState(appName); - const [errorMessage, setErrorMessage] = useState(''); - const [tenants, setUserTenants] = useState([]); - const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false); - const [isLoading, setIsLoading] = useState(false); - const [isCompleted, setIsCompleted] = useState(false); - const [indexNames, setIndexNames] = useState([]); toastManager.setTheme(theme); - function reset(): void { - setApiKey(''); - setTenantId(-1); - setUserTenants([]); - setIsLoading(false); - setIsCompleted(false); - } - - function getCurrentConfigurationStep(): ConfigurationSteps { - if (tenants.length === 0) { - return ConfigurationSteps.Initial; - } - if (!isCompleted) { - return ConfigurationSteps.UserPreferences; - } - - return ConfigurationSteps.Completed; - } - - const handleApiKeyChange = (e): void => setApiKey(e.target.value); - const handleTenantIdChange = (e): void => setTenantId(parseInt(e.target.value, 10)); - const handleIndexNameChange = (e): void => setIndexName(e.target.value); - const handleIsIngestingMetadataChange = (e): void => { - setIsIngestingMetadataOnly(e.target.checked); - }; - const handleBackButton = (): void => { - const currentConfigurationStep = getCurrentConfigurationStep(); - if (currentConfigurationStep === ConfigurationSteps.Initial) { + if (configurationStep === ConfigurationStep.Initial) { redirectToHomepage(); - } else if (currentConfigurationStep === ConfigurationSteps.UserPreferences) { - setUserTenants([]); - } else if (currentConfigurationStep === ConfigurationSteps.Completed) { - reset(); + } else if (configurationStep === ConfigurationStep.UserPreferences) { + setConfigurationStep(ConfigurationStep.Initial); + } else if (configurationStep === ConfigurationStep.Completed) { + setConfigurationStep(ConfigurationStep.Initial); } }; - const handleSubmitApiKey = (): void => { - setIsLoading(true); - fetchUserTenants( - apiKey, - (userTenants: Tenant[]) => { - if (tenantId === -1 && userTenants.length > 0) { - setTenantId(userTenants[0].id); - } - setErrorMessage(''); - setUserTenants(userTenants); - setIsLoading(false); - }, - (error: string) => { - setErrorMessage(error); - setIsLoading(false); - toastManager.show({ - id: TOAST_API_KEY_ERROR, - isError: true, - content: 'Something went wrong. Please review your form.', - }); - } - ); - }; - - const handleSubmitTenant = (): void => { - setIsLoading(true); - saveConfiguration(apiKey, tenantId, indexName, isIngestingMetadataOnly) - .then(() => { - setIsLoading(false); - setIsCompleted(true); - toastManager.destroy(TOAST_API_KEY_ERROR); - toastManager.show({ - id: TOAST_TENANT_SUCCESS, - content: 'Configured Flare Account', - }); - }) - .catch((e: any) => { - setIsLoading(false); - toastManager.show({ - id: TOAST_API_KEY_ERROR, - isError: true, - content: `Something went wrong. ${e.responseText}`, - }); - }); - }; - - function getSelectedTenantName(): string { - const filteredTenants = tenants.filter((tenant: Tenant) => tenant.id === tenantId); - if (filteredTenants.length > 0) { - return filteredTenants[0].name; - } - - return 'unknown'; - } - useEffect(() => { - if (isCompleted) { - return; - } - createFlareIndex().then(() => { - Promise.all([ - fetchApiKey(), - fetchTenantId(), - fetchIngestMetadataOnly(), - fetchCurrentIndexName(), - fetchAvailableIndexNames(), - ]).then(([key, id, ingestMetadataOnly, index, availableIndexNames]) => { + if (configurationStep === ConfigurationStep.Initial) { + Promise.all([fetchApiKey(), createFlareIndex()]).then(([key]) => { setApiKey(key); - setTenantId(id); - setIsIngestingMetadataOnly(ingestMetadataOnly); - setIndexName(index); - setIndexNames(availableIndexNames); }); - }); - }, [isCompleted]); + } + }, [configurationStep]); useEffect(() => { const container = document.getElementById('container') as HTMLDivElement; @@ -160,51 +44,37 @@ const ConfigurationScreen: FC<{ theme: string }> = ({ theme }) => { } }, [theme]); - const currentConfigurationStep = getCurrentConfigurationStep(); - return (
- +
-