Skip to content

Commit

Permalink
Severity filter
Browse files Browse the repository at this point in the history
- Fetch available severities via the rest API
- Added picker in the user preference configuration step
- Unit tests
- Logic filtering is:
	- Minimum one severity must be selected
	- If all of the severities are selected, no filter is applied in case we add new severities in the future
	- Otherwise, pass the severities selected as a filter when fetching the events
  • Loading branch information
Marc-Antoine Hinse committed Nov 28, 2024
1 parent 97c1c2e commit 683d787
Show file tree
Hide file tree
Showing 18 changed files with 399 additions and 16 deletions.
1 change: 1 addition & 0 deletions packages/flare/bin/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class PasswordKeys(Enum):
API_KEY = "api_key"
TENANT_ID = "tenant_id"
INGEST_METADATA_ONLY = "ingest_metadata_only"
SEVERITIES_FILTER = "severities_filter"


class CollectionKeys(Enum):
Expand Down
16 changes: 16 additions & 0 deletions packages/flare/bin/cron_job_ingest_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ def main(
api_key = get_api_key(storage_passwords=storage_passwords)
tenant_id = get_tenant_id(storage_passwords=storage_passwords)
ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords)
severities_filter = get_severities_filter(storage_passwords=storage_passwords)

save_last_fetched(kvstore=kvstore)
save_last_ingested_tenant_id(kvstore=kvstore, tenant_id=tenant_id)
Expand All @@ -103,6 +104,7 @@ def main(
api_key=api_key,
tenant_id=tenant_id,
ingest_metadata_only=ingest_metadata_only,
severities=severities_filter,
):
save_last_fetched(kvstore=kvstore)

Expand Down Expand Up @@ -158,6 +160,18 @@ def get_ingest_metadata_only(storage_passwords: StoragePasswords) -> bool:
)


def get_severities_filter(storage_passwords: StoragePasswords) -> list[str]:
severities_filter = get_storage_password_value(
storage_passwords=storage_passwords,
password_key=PasswordKeys.SEVERITIES_FILTER.value,
)

if severities_filter:
return severities_filter.split(",")

return []


def get_next(kvstore: KVStoreCollections, tenant_id: int) -> Optional[str]:
return get_collection_value(
kvstore=kvstore, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}"
Expand Down Expand Up @@ -281,6 +295,7 @@ def fetch_feed(
api_key: str,
tenant_id: int,
ingest_metadata_only: bool,
severities: list[str],
) -> Iterator[tuple[dict, str]]:
try:
flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id)
Expand All @@ -292,6 +307,7 @@ def fetch_feed(
next=next,
start_date=start_date,
ingest_metadata_only=ingest_metadata_only,
severities=severities,
):
yield event_next
except Exception as e:
Expand Down
13 changes: 12 additions & 1 deletion packages/flare/bin/flare.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ def fetch_feed_events(
next: Optional[str] = None,
start_date: Optional[date] = None,
ingest_metadata_only: bool,
severities: list[str],
) -> Iterator[tuple[dict, str]]:
for response in self._fetch_event_feed_metadata(
next=next,
start_date=start_date,
severities=severities,
):
event_feed = response.json()
self.logger.debug(event_feed)
Expand All @@ -71,6 +73,7 @@ def _fetch_event_feed_metadata(
*,
next: Optional[str] = None,
start_date: Optional[date] = None,
severities: list[str],
) -> Iterator[requests.Response]:
data: Dict[str, Any] = {
"from": next if next else None,
Expand All @@ -79,10 +82,13 @@ def _fetch_event_feed_metadata(
"gte": start_date.isoformat()
if start_date
else date.today().isoformat()
}
},
},
}

if len(severities):
data["severity"] = severities

for response in self.flare_client.scroll(
method="POST",
url="/firework/v4/events/tenant/_search",
Expand All @@ -107,3 +113,8 @@ def fetch_tenants(self) -> requests.Response:
return self.flare_client.get(
url="/firework/v2/me/tenants",
)

def fetch_filters_severity(self) -> requests.Response:
return self.flare_client.get(
url="/firework/v4/events/filters/severities",
)
17 changes: 17 additions & 0 deletions packages/flare/bin/flare_external_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,20 @@ def handle_POST(self) -> None:
logger.debug(f"FlareUserTenants: {response_json}")
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps(response_json))


class FlareSeverityFilters(splunk.rest.BaseRestHandler):
def handle_POST(self) -> None:
logger = Logger(class_name=__file__)
payload = self.request["payload"]
params = parse.parse_qs(payload)

if "apiKey" not in params:
raise Exception("API Key is required")

flare_api = FlareAPI(api_key=params["apiKey"][0])
response = flare_api.fetch_filters_severity()
response_json = response.json()
logger.debug(f"FlareSeverityFilters: {response_json}")
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps(response_json))
5 changes: 5 additions & 0 deletions packages/flare/src/main/resources/splunk/default/restmap.conf
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,8 @@ python.version = python3
match=/fetch_user_tenants
handler=flare_external_requests.FlareUserTenants
python.version = python3

[script:flare_external_requests_severity_filters]
match=/fetch_severity_filters
handler=flare_external_requests.FlareSeverityFilters
python.version = python3
4 changes: 4 additions & 0 deletions packages/flare/src/main/resources/splunk/default/web.conf
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,7 @@ methods=POST
[expose:flare_external_requests_user_tenants]
pattern=fetch_user_tenants
methods=POST

[expose:flare_external_requests_severity_filters]
pattern=fetch_severity_filters
methods=POST
3 changes: 3 additions & 0 deletions packages/flare/tests/bin/test_flare_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def test_flare_full_data_without_metadata(
next=None,
start_date=None,
ingest_metadata_only=True,
severities=[],
):
assert next_token == expected_return_value["next"]
events.append(event)
Expand Down Expand Up @@ -105,6 +106,7 @@ def test_flare_full_data_with_metadata(
next=None,
start_date=None,
ingest_metadata_only=False,
severities=[],
):
assert next_token == expected_return_value["next"]
events.append(event)
Expand Down Expand Up @@ -147,6 +149,7 @@ def test_flare_full_data_with_metadata_and_exception(
next=None,
start_date=None,
ingest_metadata_only=False,
severities=[],
)
)

Expand Down
3 changes: 3 additions & 0 deletions packages/flare/tests/bin/test_ingest_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ def test_fetch_feed_expect_exception() -> None:
api_key="some_key",
tenant_id=11111,
ingest_metadata_only=False,
severities=[],
):
pass

Expand Down Expand Up @@ -262,6 +263,7 @@ def test_fetch_feed_expect_feed_response(
api_key="some_key",
tenant_id=11111,
ingest_metadata_only=False,
severities=[],
):
assert next_token == next
events.append(event)
Expand Down Expand Up @@ -320,4 +322,5 @@ def test_main_expect_normal_run(
api_key="some_api_key",
tenant_id=111,
ingest_metadata_only=False,
severities=[],
)
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
import React, { FC, useEffect, useState } from 'react';
import { ConfigurationStep, Tenant } from '../models/flare';
import { ConfigurationStep, Severity, Tenant } from '../models/flare';
import Button from './Button';
import Label from './Label';
import Select from './Select';

import { APP_NAME } from '../models/constants';
import {
convertSeverityFilterToArray,
fetchAvailableIndexNames,
fetchCurrentIndexName,
fetchSeverityFilters,
fetchIngestMetadataOnly,
fetchSeveritiesFilter,
fetchTenantId,
fetchUserTenants,
getSeverityFilterValue,
saveConfiguration,
} from '../utils/setupConfiguration';
import './ConfigurationGlobalStep.css';
import './ConfigurationUserPreferencesStep.css';
import SeverityOptions from './SeverityOptions';
import Switch from './Switch';
import { ToastKeys, toastManager } from './ToastManager';
import Tooltip from './Tooltip';
Expand All @@ -28,6 +33,8 @@ const ConfigurationUserPreferencesStep: FC<{
}> = ({ show, configurationStep, apiKey, onNavigateBackClick, onUserPreferencesSaved }) => {
const [tenantId, setTenantId] = useState<number | undefined>(undefined);
const [tenants, setUserTenants] = useState<Tenant[]>([]);
const [selectedSeverities, setSelectedSeverities] = useState<Severity[]>([]);
const [severities, setSeverities] = useState<Severity[]>([]);
const [indexName, setIndexName] = useState('');
const [indexNames, setIndexNames] = useState<string[]>([]);
const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false);
Expand All @@ -41,7 +48,13 @@ const ConfigurationUserPreferencesStep: FC<{
const handleSubmitUserPreferences = (): void => {
setIsLoading(true);

saveConfiguration(apiKey, Number(tenantId), indexName, isIngestingMetadataOnly)
saveConfiguration(
apiKey,
Number(tenantId),
indexName,
isIngestingMetadataOnly,
getSeverityFilterValue(selectedSeverities, severities)
)
.then(() => {
setIsLoading(false);
toastManager.destroy(ToastKeys.ERROR);
Expand Down Expand Up @@ -69,17 +82,33 @@ const ConfigurationUserPreferencesStep: FC<{
fetchCurrentIndexName(),
fetchUserTenants(apiKey),
fetchAvailableIndexNames(),
fetchSeverityFilters(apiKey),
fetchSeveritiesFilter(),
])
.then(([id, ingestMetadataOnly, index, userTenants, availableIndexNames]) => {
setTenantId(id);
setIsIngestingMetadataOnly(ingestMetadataOnly);
setIndexName(index);
if (id === -1 && userTenants.length > 0) {
setTenantId(userTenants[0].id);
.then(
([
id,
ingestMetadataOnly,
index,
userTenants,
availableIndexNames,
allSeverities,
severitiesFilter,
]) => {
setTenantId(id);
setIsIngestingMetadataOnly(ingestMetadataOnly);
setIndexName(index);
if (id === -1 && userTenants.length > 0) {
setTenantId(userTenants[0].id);
}
setUserTenants(userTenants);
setIndexNames(availableIndexNames);
setSeverities(allSeverities);
setSelectedSeverities(
convertSeverityFilterToArray(severitiesFilter, allSeverities)
);
}
setUserTenants(userTenants);
setIndexNames(availableIndexNames);
})
)
.catch(() => {
toastManager.show({
id: ToastKeys.ERROR,
Expand All @@ -93,11 +122,13 @@ const ConfigurationUserPreferencesStep: FC<{
setIndexNames([]);
setUserTenants([]);
setIsLoading(false);
setSeverities([]);
setSelectedSeverities([]);
}
}, [configurationStep, apiKey]);

const isFormValid = (): boolean => {
return tenantId !== undefined;
return tenantId !== undefined && selectedSeverities.length > 0;
};

return (
Expand Down Expand Up @@ -129,6 +160,31 @@ const ConfigurationUserPreferencesStep: FC<{
})}
</Select>
</div>
<div className="form-item">
<div className="label-tooltip">
<Label>Severity filter</Label>
<Tooltip>
<div>
Select the minimal alert severity to ignore less critical events
associated with this identifier.
<br />
<br />
{'To learn more about severities see '}
<a
target="_blank"
href="https://docs.flare.io/understand-event-severity"
>
Understand Severity Scoring.
</a>
</div>
</Tooltip>
</div>
<SeverityOptions
setSelectedSeverities={setSelectedSeverities}
severities={severities}
selectedSeverities={selectedSeverities}
/>
</div>
<div className="form-item">
<div className="label-tooltip">
<Label>Basic event ingestion</Label>
Expand Down
28 changes: 28 additions & 0 deletions packages/react-components/src/components/SeverityOption.css
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
.toggle {
position: relative;
width: 1rem;
height: 1rem;
display: inline-block;
z-index: 2;
}

.toggle input {
opacity: 0;
width: 0;
height: 0;
}

.dot {
position: absolute;
cursor: pointer;
top: 0;
left: 0;
right: 0;
bottom: 0;
height: 1rem;
width: 1rem;
border-radius: 50%;
border-width: 1px;
border-style: solid;
display: inline-block;
}
Loading

0 comments on commit 683d787

Please sign in to comment.