Skip to content

Commit

Permalink
Severity and categories filter
Browse files Browse the repository at this point in the history
- Moved back the configuration steps business logic into their own component
- Added support for severity and source types in the fetch event requests
- Created components for severity options and a source type list
- Created a constants file
- Block the user from going to the next step if the form isn't valid
  • Loading branch information
Marc-Antoine Hinse committed Nov 26, 2024
1 parent 7ee82cc commit 97d9b2a
Show file tree
Hide file tree
Showing 35 changed files with 1,129 additions and 311 deletions.
2 changes: 2 additions & 0 deletions packages/flare/bin/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ class PasswordKeys(Enum):
API_KEY = "api_key"
TENANT_ID = "tenant_id"
INGEST_METADATA_ONLY = "ingest_metadata_only"
SEVERITIES_FILTER = "severities_filter"
SOURCE_TYPES_FILTER = "source_types_filter"


class CollectionKeys(Enum):
Expand Down
34 changes: 33 additions & 1 deletion packages/flare/bin/cron_job_ingest_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ def main(logger: Logger, app: client.Application) -> None:
api_key = get_api_key(app=app)
tenant_id = get_tenant_id(app=app)
ingest_metadata_only = get_ingest_metadata_only(app=app)
severities_filter = get_severities_filter(app=app)
source_types_filter = get_source_types_filter(app=app)

save_last_fetched(app=app)
save_last_ingested_tenant_id(app=app, tenant_id=tenant_id)
Expand All @@ -50,6 +52,8 @@ def main(logger: Logger, app: client.Application) -> None:
api_key=api_key,
tenant_id=tenant_id,
ingest_metadata_only=ingest_metadata_only,
severities=severities_filter,
source_types=source_types_filter,
):
save_last_fetched(app=app)

Expand Down Expand Up @@ -104,6 +108,28 @@ def get_ingest_metadata_only(app: client.Application) -> bool:
)


def get_severities_filter(app: client.Application) -> list[str]:
severities_filter = get_storage_password_value(
app=app, password_key=PasswordKeys.SEVERITIES_FILTER.value
)

if severities_filter:
return severities_filter.split(",")

return []


def get_source_types_filter(app: client.Application) -> list[str]:
source_types_filter = get_storage_password_value(
app=app, password_key=PasswordKeys.SOURCE_TYPES_FILTER.value
)

if source_types_filter:
return source_types_filter.split(",")

return []


def get_next(app: client.Application, tenant_id: int) -> Optional[str]:
return get_collection_value(
app=app, key=f"{CollectionKeys.get_next_token(tenantId=tenant_id)}"
Expand Down Expand Up @@ -222,6 +248,8 @@ def fetch_feed(
api_key: str,
tenant_id: int,
ingest_metadata_only: bool,
severities: list[str],
source_types: list[str],
) -> Iterator[tuple[dict, str]]:
try:
flare_api = FlareAPI(api_key=api_key, tenant_id=tenant_id)
Expand All @@ -230,7 +258,11 @@ def fetch_feed(
start_date = get_start_date(app=app)
logger.info(f"Fetching {tenant_id=}, {next=}, {start_date=}")
for event_next in flare_api.fetch_feed_events(
next=next, start_date=start_date, ingest_metadata_only=ingest_metadata_only
next=next,
start_date=start_date,
ingest_metadata_only=ingest_metadata_only,
severities=severities,
source_types=source_types,
):
yield event_next
except Exception as e:
Expand Down
25 changes: 24 additions & 1 deletion packages/flare/bin/flare.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,14 @@ def fetch_feed_events(
next: Optional[str] = None,
start_date: Optional[date] = None,
ingest_metadata_only: bool,
severities: list[str],
source_types: list[str],
) -> Iterator[tuple[dict, str]]:
for response in self._fetch_event_feed_metadata(
next=next,
start_date=start_date,
severities=severities,
source_types=source_types,
):
event_feed = response.json()
self.logger.debug(event_feed)
Expand All @@ -71,6 +75,8 @@ def _fetch_event_feed_metadata(
*,
next: Optional[str] = None,
start_date: Optional[date] = None,
severities: list[str],
source_types: list[str],
) -> Iterator[requests.Response]:
data: Dict[str, Any] = {
"from": next if next else None,
Expand All @@ -79,7 +85,9 @@ def _fetch_event_feed_metadata(
"gte": start_date.isoformat()
if start_date
else date.today().isoformat()
}
},
"severity": severities if len(severities) > 0 else None,
"types": source_types if len(source_types) > 0 else None,
},
}

Expand All @@ -98,7 +106,22 @@ def _fetch_full_event_from_uid(self, *, uid: str) -> dict:
self.logger.debug(event)
return event

def fetch_api_key_validation(self) -> requests.Response:
return self.flare_client.get(
url="/tokens/test",
)

def fetch_tenants(self) -> requests.Response:
return self.flare_client.get(
url="/firework/v2/me/tenants",
)

def fetch_filters_severity(self) -> requests.Response:
return self.flare_client.get(
url="/firework/v4/events/filters/severities",
)

def fetch_filters_source_types(self) -> requests.Response:
return self.flare_client.get(
url="/firework/v4/events/filters/types",
)
56 changes: 52 additions & 4 deletions packages/flare/bin/flare_external_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,20 @@
from logger import Logger


class FlareValidateApiKey(splunk.rest.BaseRestHandler):
def handle_POST(self) -> None:
payload = self.request["payload"]
params = parse.parse_qs(payload)

if "apiKey" in params:
flare_api = FlareAPI(api_key=params["apiKey"][0])
flare_api.fetch_api_key_validation()
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps({}))
else:
raise Exception("API Key is required")


class FlareUserTenants(splunk.rest.BaseRestHandler):
def handle_POST(self) -> None:
logger = Logger(class_name=__file__)
Expand All @@ -20,10 +34,44 @@ def handle_POST(self) -> None:

if "apiKey" in params:
flare_api = FlareAPI(api_key=params["apiKey"][0])
user_tenants_response = flare_api.fetch_tenants()
tenants_response = user_tenants_response.json()
logger.debug(tenants_response)
response = flare_api.fetch_tenants()
response_json = response.json()
logger.debug(f"FlareUserTenants: {response_json}")
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps(response_json))
else:
raise Exception("API Key is required")


class FlareFiltersSeverity(splunk.rest.BaseRestHandler):
def handle_POST(self) -> None:
logger = Logger(class_name=__file__)
payload = self.request["payload"]
params = parse.parse_qs(payload)

if "apiKey" in params:
flare_api = FlareAPI(api_key=params["apiKey"][0])
response = flare_api.fetch_filters_severity()
response_json = response.json()
logger.debug(f"FlareFiltersSeverity: {response_json}")
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps(response_json))
else:
raise Exception("API Key is required")


class FlareFiltersSourceTypes(splunk.rest.BaseRestHandler):
def handle_POST(self) -> None:
logger = Logger(class_name=__file__)
payload = self.request["payload"]
params = parse.parse_qs(payload)

if "apiKey" in params:
flare_api = FlareAPI(api_key=params["apiKey"][0])
response = flare_api.fetch_filters_source_types()
response_json = response.json()
logger.debug(f"FlareFiltersSourceTypes: {response_json}")
self.response.setHeader("Content-Type", "application/json")
self.response.write(json.dumps(tenants_response))
self.response.write(json.dumps(response_json))
else:
raise Exception("API Key is required")
15 changes: 15 additions & 0 deletions packages/flare/src/main/resources/splunk/default/restmap.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,19 @@
[script:flare_external_requests_api_key_validation]
match=/fetch_api_key_validation
handler=flare_external_requests.FlareValidateApiKey
python.version = python3

[script:flare_external_requests_user_tenants]
match=/fetch_user_tenants
handler=flare_external_requests.FlareUserTenants
python.version = python3

[script:flare_external_requests_filters_severities]
match=/fetch_filters_severities
handler=flare_external_requests.FlareFiltersSeverity
python.version = python3

[script:flare_external_requests_filters_source_types]
match=/fetch_filters_source_types
handler=flare_external_requests.FlareFiltersSourceTypes
python.version = python3
12 changes: 12 additions & 0 deletions packages/flare/src/main/resources/splunk/default/web.conf
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
[expose:flare_external_requests_api_key_validation]
pattern=fetch_api_key_validation
methods=POST

[expose:flare_external_requests_user_tenants]
pattern=fetch_user_tenants
methods=POST

[expose:flare_external_requests_filters_severities]
pattern=fetch_filters_severities
methods=POST

[expose:flare_external_requests_filters_source_types]
pattern=fetch_filters_source_types
methods=POST
18 changes: 15 additions & 3 deletions packages/flare/tests/bin/test_flare_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ def test_flare_full_data_without_metadata(

events: list[dict] = []
for event, next_token in flare_api.fetch_feed_events(
next=None, start_date=None, ingest_metadata_only=True
next=None,
start_date=None,
ingest_metadata_only=True,
severities=[],
source_types=[],
):
assert next_token == expected_return_value["next"]
events.append(event)
Expand Down Expand Up @@ -100,7 +104,11 @@ def test_flare_full_data_with_metadata(

events: list[dict] = []
for event, next_token in flare_api.fetch_feed_events(
next=None, start_date=None, ingest_metadata_only=False
next=None,
start_date=None,
ingest_metadata_only=False,
severities=[],
source_types=[],
):
assert next_token == expected_return_value["next"]
events.append(event)
Expand Down Expand Up @@ -139,7 +147,11 @@ def test_flare_full_data_with_metadata_and_exception(

with pytest.raises(KeyError, match="metadata"):
for _, _ in flare_api.fetch_feed_events(
next=None, start_date=None, ingest_metadata_only=False
next=None,
start_date=None,
ingest_metadata_only=False,
severities=[],
source_types=[],
):
pass

Expand Down
6 changes: 6 additions & 0 deletions packages/flare/tests/bin/test_ingest_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ def test_fetch_feed_expect_exception() -> None:
api_key="some_key",
tenant_id=11111,
ingest_metadata_only=False,
severities=[],
source_types=[],
):
pass

Expand Down Expand Up @@ -258,6 +260,8 @@ def test_fetch_feed_expect_feed_response(
api_key="some_key",
tenant_id=11111,
ingest_metadata_only=False,
severities=[],
source_types=[],
):
assert next_token == next
events.append(event)
Expand Down Expand Up @@ -314,4 +318,6 @@ def test_main_expect_normal_run(
api_key="some_api_key",
tenant_id=111,
ingest_metadata_only=False,
severities=[],
source_types=[],
)
1 change: 1 addition & 0 deletions packages/react-components/src/ConfigurationScreen.css
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
width: 800px;
gap: 1.5rem;
align-self: center;
padding-bottom: 2rem;
}

.content-step {
Expand Down
Loading

0 comments on commit 97d9b2a

Please sign in to comment.