From b8cb0ae8adb6999ed064f6f0c26c6ab34ac89f65 Mon Sep 17 00:00:00 2001 From: Amardeepsingh Siglani Date: Tue, 21 May 2024 22:39:58 -0700 Subject: [PATCH] Updated get findings & alerts to use duration filter and start showing results as they come in (#1031) * updated get findings & alerts to use duration filter and stream results Signed-off-by: Amardeepsingh Siglani * refactored code to use AbortController Signed-off-by: Amardeepsingh Siglani * minor UI tweaks Signed-off-by: Amardeepsingh Siglani --------- Signed-off-by: Amardeepsingh Siglani --- .../components/AlertFlyout/AlertFlyout.tsx | 9 +- .../pages/Alerts/containers/Alerts/Alerts.tsx | 39 +++++--- .../Alerts/__snapshots__/Alerts.test.tsx.snap | 85 +++++++++++++++- .../containers/CorrelationsContainer.tsx | 14 ++- .../CorrelationsTable/CorrelationsTable.tsx | 3 +- .../components/FindingDetailsFlyout.tsx | 16 +++- .../FindingsTable/FindingsTable.tsx | 7 +- .../Findings/containers/Findings/Findings.tsx | 78 ++++++++------- public/pages/Findings/models/interfaces.ts | 11 --- public/pages/Main/Main.tsx | 1 + .../Overview/containers/Overview/Overview.tsx | 49 ++++++++-- .../Overview/models/OverviewViewModel.ts | 87 +++++++++++++---- public/react-graph-vis.d.ts | 8 +- public/services/AlertsService.ts | 4 +- public/services/FindingsService.ts | 8 +- public/store/AlertsStore.ts | 20 +++- public/store/CorrelationsStore.ts | 96 ++++++++++++------- public/store/FindingsStore.ts | 92 +++++++++++++----- public/utils/helpers.tsx | 13 ++- server/routes/AlertRoutes.ts | 2 + server/routes/FindingsRoutes.ts | 2 + server/services/AlertService.ts | 4 +- types/Alert.ts | 2 + types/Correlations.ts | 1 - types/Overview.ts | 2 +- types/index.ts | 1 + types/shared.ts | 27 ++++++ 27 files changed, 505 insertions(+), 176 deletions(-) create mode 100644 types/shared.ts diff --git a/public/pages/Alerts/components/AlertFlyout/AlertFlyout.tsx b/public/pages/Alerts/components/AlertFlyout/AlertFlyout.tsx index f1da52711..73f98ad2d 100644 --- a/public/pages/Alerts/components/AlertFlyout/AlertFlyout.tsx +++ b/public/pages/Alerts/components/AlertFlyout/AlertFlyout.tsx @@ -17,7 +17,7 @@ import { EuiSpacer, EuiTitle, } from '@elastic/eui'; -import { AlertItem, RuleSource } from '../../../../../server/models/interfaces'; +import { RuleSource } from '../../../../../server/models/interfaces'; import React from 'react'; import { ContentPanel } from '../../../../components/ContentPanel'; import { ALERT_STATE, DEFAULT_EMPTY_DATA, ROUTES } from '../../../../utils/constants'; @@ -30,10 +30,9 @@ import { } from '../../../../utils/helpers'; import { IndexPatternsService, OpenSearchService } from '../../../../services'; import { parseAlertSeverityToOption } from '../../../CreateDetector/components/ConfigureAlerts/utils/helpers'; -import { Finding } from '../../../Findings/models/interfaces'; import { NotificationsStart } from 'opensearch-dashboards/public'; import { DataStore } from '../../../../store/DataStore'; -import { Detector } from '../../../../../types'; +import { AlertItem, Detector, Finding } from '../../../../../types'; export interface AlertFlyoutProps { alertItem: AlertItem; @@ -135,7 +134,7 @@ export class AlertFlyout extends React.Component + render: (id: string, finding: any) => ( { @@ -159,7 +158,7 @@ export class AlertFlyout extends React.Component - {`${(id as string).slice(0, 7)}...`} + {id.length > 7 ? `${id.slice(0, 7)}...` : id} ) || DEFAULT_EMPTY_DATA, }, diff --git a/public/pages/Alerts/containers/Alerts/Alerts.tsx b/public/pages/Alerts/containers/Alerts/Alerts.tsx index bcb98c571..511579816 100644 --- a/public/pages/Alerts/containers/Alerts/Alerts.tsx +++ b/public/pages/Alerts/containers/Alerts/Alerts.tsx @@ -48,6 +48,7 @@ import { capitalizeFirstLetter, createSelectComponent, errorNotificationToast, + getDuration, renderTime, renderVisualization, successNotificationToast, @@ -67,7 +68,7 @@ export interface AlertsProps extends RouteComponentProps, DataSourceProps { notifications: NotificationsStart; indexPatternService: IndexPatternsService; match: match<{ detectorId: string }>; - dateTimeFilter?: DateTimeFilter; + dateTimeFilter: DateTimeFilter; setDateTimeFilter?: Function; } @@ -93,6 +94,7 @@ const groupByOptions = [ export class Alerts extends Component { static contextType = CoreServicesContext; + private abortControllers: AbortController[] = []; constructor(props: AlertsProps) { super(props); @@ -300,32 +302,39 @@ export class Alerts extends Component { this.onRefresh(); } - async getAlerts() { - this.setState({ loading: true }); - const { detectorService, notifications } = this.props; + componentWillUnmount(): void { + this.abortPendingGetAlerts(); + } + + async getAlerts(abort: AbortSignal) { + this.setState({ loading: true, alerts: [] }); + const { detectorService, notifications, dateTimeFilter } = this.props; const { detectors } = this.state; try { const detectorsRes = await detectorService.getDetectors(); + const duration = getDuration(dateTimeFilter); if (detectorsRes.ok) { + this.setState({ detectors: detectors }); const detectorIds = detectorsRes.response.hits.hits.map((hit) => { detectors[hit._id] = { ...hit._source, id: hit._id }; return hit._id; }); - let alerts: AlertItem[] = []; const detectorId = this.props.match.params['detectorId']; for (let id of detectorIds) { if (!detectorId || detectorId === id) { - const detectorAlerts = await DataStore.alerts.getAlertsByDetector( + await DataStore.alerts.getAlertsByDetector( id, - detectors[id].name + detectors[id].name, + abort, + duration, + (alerts) => { + this.setState({ alerts: [...this.state.alerts, ...alerts]}) + } ); - alerts = alerts.concat(detectorAlerts); } } - - this.setState({ alerts: alerts, detectors: detectors }); } else { errorNotificationToast(notifications, 'retrieve', 'detectors', detectorsRes.error); } @@ -372,8 +381,16 @@ export class Alerts extends Component { }); }; + private abortPendingGetAlerts() { + this.abortControllers.forEach(controller => controller.abort()); + this.abortControllers = []; + } + onRefresh = async () => { - this.getAlerts(); + this.abortPendingGetAlerts(); + const abortController = new AbortController(); + this.abortControllers.push(abortController); + this.getAlerts(abortController.signal); renderVisualization(this.generateVisualizationSpec(this.state.filteredAlerts), 'alerts-view'); }; diff --git a/public/pages/Alerts/containers/Alerts/__snapshots__/Alerts.test.tsx.snap b/public/pages/Alerts/containers/Alerts/__snapshots__/Alerts.test.tsx.snap index 09dba1179..f05f80fc1 100644 --- a/public/pages/Alerts/containers/Alerts/__snapshots__/Alerts.test.tsx.snap +++ b/public/pages/Alerts/containers/Alerts/__snapshots__/Alerts.test.tsx.snap @@ -1182,6 +1182,24 @@ exports[` spec renders the component 1`] = ` itemId={[Function]} items={Array []} loading={true} + message={ + + + No alerts. + + Adjust the time range to see more results. +

+ } + /> + } pagination={true} responsive={true} search={ @@ -1745,7 +1763,24 @@ exports[` spec renders the component 1`] = ` itemId={[Function]} items={Array []} loading={true} - noItemsMessage="No items found" + noItemsMessage={ + + + No alerts. + + Adjust the time range to see more results. +

+ } + /> + } onChange={[Function]} pagination={ Object { @@ -2408,7 +2443,53 @@ exports[` spec renders the component 1`] = ` - No items found + + + No alerts. + + Adjust the time range to see more results. +

+ } + > +
+ + + +
+

+ + No alerts. + + Adjust the time range to see more results. +

+
+ + + +
+ diff --git a/public/pages/Correlations/containers/CorrelationsContainer.tsx b/public/pages/Correlations/containers/CorrelationsContainer.tsx index 5ab4a897d..af6b2a9c9 100644 --- a/public/pages/Correlations/containers/CorrelationsContainer.tsx +++ b/public/pages/Correlations/containers/CorrelationsContainer.tsx @@ -7,6 +7,7 @@ import { CorrelationGraphData, DataSourceProps, DateTimeFilter, + FindingItemType, } from '../../../../types'; import React from 'react'; import { RouteComponentProps } from 'react-router-dom'; @@ -50,12 +51,13 @@ import { import { CorrelationGraph } from '../components/CorrelationGraph'; import { FindingCard } from '../components/FindingCard'; import { DataStore } from '../../../store/DataStore'; -import { FindingItemType } from '../../Findings/containers/Findings/Findings'; import datemath from '@elastic/datemath'; import { ruleSeverity } from '../../Rules/utils/constants'; import { renderToStaticMarkup } from 'react-dom/server'; import { Network } from 'react-graph-vis'; import { getLogTypeLabel } from '../../LogTypes/utils/helpers'; +import { NotificationsStart } from 'opensearch-dashboards/public'; +import { errorNotificationToast } from '../../../utils/helpers'; interface CorrelationsProps extends RouteComponentProps< @@ -67,6 +69,7 @@ interface CorrelationsProps setDateTimeFilter?: Function; dateTimeFilter?: DateTimeFilter; onMount: () => void; + notifications: NotificationsStart | null; } interface SpecificFindingCorrelations { @@ -237,8 +240,13 @@ export class Correlations extends React.Component { + private abortGetFindingsControllers: AbortController[] = []; + constructor(props: FindingDetailsFlyoutProps) { super(props); const relatedDocuments: FindingDocumentItem[] = this.getRelatedDocuments(); @@ -121,12 +122,21 @@ export default class FindingDetailsFlyout extends Component< }; } + componentWillUnmount(): void { + this.abortGetFindingsControllers.forEach(controller => { + controller.abort(); + }) + this.abortGetFindingsControllers = []; + } + getCorrelations = async () => { const { id, detector } = this.props.finding; let allFindings = this.props.findings; if (this.props.shouldLoadAllFindings) { // if findings come from the alerts fly-out, we need to get all the findings to match those with the correlations - allFindings = await DataStore.findings.getAllFindings(); + const abortController = new AbortController(); + this.abortGetFindingsControllers.push(abortController); + allFindings = await DataStore.findings.getAllFindings(abortController.signal); } DataStore.correlations.getCorrelationRules().then((correlationRules) => { diff --git a/public/pages/Findings/components/FindingsTable/FindingsTable.tsx b/public/pages/Findings/components/FindingsTable/FindingsTable.tsx index cc9ec9713..05d771ffa 100644 --- a/public/pages/Findings/components/FindingsTable/FindingsTable.tsx +++ b/public/pages/Findings/components/FindingsTable/FindingsTable.tsx @@ -30,14 +30,13 @@ import { IndexPatternsService, CorrelationService, } from '../../../../services'; -import { Finding } from '../../models/interfaces'; import CreateAlertFlyout from '../CreateAlertFlyout'; import { NotificationChannelTypeOptions } from '../../../CreateDetector/components/ConfigureAlerts/models/interfaces'; -import { FindingItemType } from '../../containers/Findings/Findings'; import { parseAlertSeverityToOption } from '../../../CreateDetector/components/ConfigureAlerts/utils/helpers'; import { RuleSource } from '../../../../../server/models/interfaces'; import { DataStore } from '../../../../store/DataStore'; import { getSeverityColor } from '../../../Correlations/utils/constants'; +import { Finding, FindingItemType } from '../../../../../types'; interface FindingsTableProps extends RouteComponentProps { detectorService: DetectorsService; @@ -177,13 +176,13 @@ export default class FindingsTable extends Component + render: (id: string, finding) => ( DataStore.findings.openFlyout(finding, this.state.filteredFindings)} data-test-subj={'finding-details-flyout-button'} > - {`${(id as string).slice(0, 7)}...`} + {id.length > 7 ? `${id.slice(0, 7)}...` : id} ) || DEFAULT_EMPTY_DATA, }, diff --git a/public/pages/Findings/containers/Findings/Findings.tsx b/public/pages/Findings/containers/Findings/Findings.tsx index a206dd71e..d9f197107 100644 --- a/public/pages/Findings/containers/Findings/Findings.tsx +++ b/public/pages/Findings/containers/Findings/Findings.tsx @@ -37,7 +37,6 @@ import { TimeUnit, } from '../../../Overview/utils/helpers'; import { CoreServicesContext } from '../../../../components/core_services'; -import { Finding } from '../../models/interfaces'; import { getNotificationChannels, parseNotificationChannelsToOptions, @@ -47,17 +46,19 @@ import { errorNotificationToast, renderVisualization, getPlugins, + getDuration, } from '../../../../utils/helpers'; -import { DetectorHit, RuleSource } from '../../../../../server/models/interfaces'; +import { RuleSource } from '../../../../../server/models/interfaces'; import { NotificationsStart } from 'opensearch-dashboards/public'; import { ChartContainer } from '../../../../components/Charts/ChartContainer'; import { DataStore } from '../../../../store/DataStore'; import { DurationRange } from '@elastic/eui/src/components/date_picker/types'; import { - CorrelationFinding, DataSourceProps, FeatureChannelList, DateTimeFilter, + FindingItemType, + DetectorHit } from '../../../../../types'; interface FindingsProps extends RouteComponentProps, DataSourceProps { @@ -92,10 +93,6 @@ interface FindingVisualizationData { ruleSeverity: string; } -export type FindingItemType = Finding & { detector: DetectorHit } & { - correlations: CorrelationFinding[]; -}; - type FindingsGroupByType = 'logType' | 'ruleSeverity'; export const groupByOptions = [ @@ -106,6 +103,8 @@ export const groupByOptions = [ class Findings extends Component { static contextType = CoreServicesContext; + private abortGetFindingsControllers: AbortController[] = []; + constructor(props: FindingsProps) { super(props); @@ -146,56 +145,61 @@ class Findings extends Component { this.onRefresh(); }; + componentWillUnmount(): void { + this.abortGetFindings(); + } + onRefresh = async () => { - await this.getFindings(); await this.getNotificationChannels(); await this.getPlugins(); + await this.getFindings(); renderVisualization(this.generateVisualizationSpec(), 'findings-view'); }; + onStreamingFindings = async (findings: FindingItemType[]) => { + const ruleIds = new Set(); + findings.forEach((finding) => { + finding.queries.forEach((rule) => ruleIds.add(rule.id)); + }); + + await this.getRules(Array.from(ruleIds)); + this.setState({ findings: [...this.state.findings, ...findings] }); + } + + abortGetFindings = () => { + this.abortGetFindingsControllers.forEach(controller => { + controller.abort(); + }); + } + getFindings = async () => { - this.setState({ loading: true }); - const { detectorService, notifications } = this.props; + this.abortGetFindings(); + this.setState({ loading: true, findings: [] }); + const { detectorService, notifications, dateTimeFilter } = this.props; + const abortController = new AbortController(); + this.abortGetFindingsControllers.push(abortController); try { - const ruleIds = new Set(); - let findings: FindingItemType[] = []; - const detectorId = this.props.match.params['detectorId']; + const duration = dateTimeFilter ? getDuration(dateTimeFilter) : undefined; // Not looking for findings from specific detector if (!detectorId) { - findings = await DataStore.findings.getAllFindings(); + await DataStore.findings.getAllFindings(abortController.signal, duration, this.onStreamingFindings); } else { // get findings for a detector - const detectorFindings = await DataStore.findings.getFindingsPerDetector(detectorId); const getDetectorResponse = await detectorService.getDetectorWithId(detectorId); if (getDetectorResponse.ok) { - const detector = getDetectorResponse.response.detector; - findings = detectorFindings.map((finding) => { - return { - ...finding, - detectorName: detector.name, - logType: detector.detector_type, - detector: { - _id: getDetectorResponse.response._id, - _source: detector, - _index: '', - }, - correlations: [], - }; - }); + const detectorHit: DetectorHit = { + _id: getDetectorResponse.response._id, + _index: '', + _source: getDetectorResponse.response.detector + } + await DataStore.findings.getFindingsPerDetector(detectorId, detectorHit, abortController.signal, duration, this.onStreamingFindings); } else { errorNotificationToast(notifications, 'retrieve', 'findings', getDetectorResponse.error); } } - - findings.forEach((finding) => { - finding.queries.forEach((rule) => ruleIds.add(rule.id)); - }); - - await this.getRules(Array.from(ruleIds)); - this.setState({ findings }); } catch (e) { errorNotificationToast(notifications, 'retrieve', 'findings', e); } @@ -209,7 +213,7 @@ class Findings extends Component { _id: ruleIds, }); - const allRules: { [id: string]: RuleSource } = {}; + const allRules: { [id: string]: RuleSource } = { ...this.state.rules }; rules.forEach((hit) => (allRules[hit._id] = hit._source)); this.setState({ rules: allRules }); diff --git a/public/pages/Findings/models/interfaces.ts b/public/pages/Findings/models/interfaces.ts index 353350731..65cddf08a 100644 --- a/public/pages/Findings/models/interfaces.ts +++ b/public/pages/Findings/models/interfaces.ts @@ -3,17 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -export interface Finding { - id: string; - detectorId: string; - document_list: FindingDocument[]; - index: string; - queries: Query[]; - related_doc_ids: string[]; - timestamp: number; - detectionType: string; -} - export interface Query { id: string; name: string; diff --git a/public/pages/Main/Main.tsx b/public/pages/Main/Main.tsx index 381c1e4e2..d1629cd89 100644 --- a/public/pages/Main/Main.tsx +++ b/public/pages/Main/Main.tsx @@ -674,6 +674,7 @@ export default class Main extends Component { dateTimeFilter={this.state.dateTimeFilter} setDateTimeFilter={this.setDateTimeFilter} dataSource={selectedDataSource} + notifications={core?.notifications} /> ); }} diff --git a/public/pages/Overview/containers/Overview/Overview.tsx b/public/pages/Overview/containers/Overview/Overview.tsx index 863833e2e..365dc0d40 100644 --- a/public/pages/Overview/containers/Overview/Overview.tsx +++ b/public/pages/Overview/containers/Overview/Overview.tsx @@ -14,7 +14,7 @@ import { EuiSpacer, EuiButton, } from '@elastic/eui'; -import React, { useContext, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useMemo, useState } from 'react'; import { BREADCRUMBS, DEFAULT_DATE_RANGE, @@ -61,15 +61,29 @@ export const Overview: React.FC = (props) => { const context = useContext(CoreServicesContext); const saContext = useContext(SecurityAnalyticsContext); + const [abortController, setControllers] = useState>([]); + const fireAbortSignals = useCallback(() => { + abortController.forEach(controller => { + controller.abort(); + }); + }, [abortController]); + + // This essentially makes sure we fire abort signals on the component unmount + useEffect(() => { + return fireAbortSignals; + }, [fireAbortSignals]); - const updateState = (overviewViewModel: OverviewViewModel) => { + const updateState = (overviewViewModel: OverviewViewModel, modelLoadingComplete: boolean) => { setState({ ...state, overviewViewModel: { ...overviewViewModel }, }); - setLoading(false); }; + const onLoadingComplete = (_overviewViewModel: OverviewViewModel, modelLoadingComplete: boolean) => { + setLoading(!modelLoadingComplete); + } + const overviewViewModelActor = useMemo( () => new OverviewViewModelActor(saContext?.services, context?.notifications!), [saContext?.services, context] @@ -77,12 +91,15 @@ export const Overview: React.FC = (props) => { useEffect(() => { context?.chrome.setBreadcrumbs([BREADCRUMBS.SECURITY_ANALYTICS, BREADCRUMBS.OVERVIEW]); - overviewViewModelActor.registerRefreshHandler(updateState); + overviewViewModelActor.registerRefreshHandler(updateState, true /* allowPartialResults */); + overviewViewModelActor.registerRefreshHandler(onLoadingComplete, false /* allowPartialResults */); }, []); useEffect(() => { + const abortController = new AbortController(); + const updateModel = async () => { - await overviewViewModelActor.onRefresh(dateTimeFilter.startTime, dateTimeFilter.endTime); + await overviewViewModelActor.onRefresh(dateTimeFilter.startTime, dateTimeFilter.endTime, abortController.signal); if (!initialLoadingFinished) { setInitialLoadingFinished(true); @@ -90,6 +107,10 @@ export const Overview: React.FC = (props) => { }; updateModel(); + + return () => { + abortController.abort() + } }, [dateTimeFilter.startTime, dateTimeFilter.endTime]); useEffect(() => { @@ -122,13 +143,18 @@ export const Overview: React.FC = (props) => { setRecentlyUsedRanges(usedRanges); }; - const onRefresh = async () => { + const onRefresh = async (signal: AbortSignal) => { setLoading(true); - await overviewViewModelActor.onRefresh(dateTimeFilter.startTime, dateTimeFilter.endTime); + await overviewViewModelActor.onRefresh(dateTimeFilter.startTime, dateTimeFilter.endTime, signal); }; useEffect(() => { - onRefresh(); + const abortController = new AbortController(); + onRefresh(abortController.signal); + + return () => { + abortController.abort(); + } }, [props.dataSource]); const onButtonClick = () => setIsPopoverOpen((isPopoverOpen) => !isPopoverOpen); @@ -170,7 +196,12 @@ export const Overview: React.FC = (props) => { recentlyUsedRanges={recentlyUsedRanges} isLoading={loading} onTimeChange={onTimeChange} - onRefresh={onRefresh} + onRefresh={() => { + const abortController = new AbortController(); + fireAbortSignals(); + setControllers([abortController]); + onRefresh(abortController.signal); + }} updateButtonProps={{ fill: false }} /> diff --git a/public/pages/Overview/models/OverviewViewModel.ts b/public/pages/Overview/models/OverviewViewModel.ts index 15be20040..8f507cd94 100644 --- a/public/pages/Overview/models/OverviewViewModel.ts +++ b/public/pages/Overview/models/OverviewViewModel.ts @@ -7,11 +7,12 @@ import { BrowserServices } from '../../../models/interfaces'; import { RuleSource } from '../../../../server/models/interfaces'; import { DEFAULT_DATE_RANGE, DEFAULT_EMPTY_DATA } from '../../../utils/constants'; import { NotificationsStart } from 'opensearch-dashboards/public'; -import { errorNotificationToast, isThreatIntelQuery } from '../../../utils/helpers'; +import { errorNotificationToast, getDuration, isThreatIntelQuery } from '../../../utils/helpers'; import dateMath from '@elastic/datemath'; import moment from 'moment'; import { DataStore } from '../../../store/DataStore'; import { + DetectorHit, Finding, OverviewAlertItem, OverviewFindingItem, @@ -25,7 +26,8 @@ export class OverviewViewModelActor { findings: [], alerts: [], }; - private refreshHandlers: OverviewViewModelRefreshHandler[] = []; + private partialUpdateHandlers: OverviewViewModelRefreshHandler[] = []; + private fullUpdateHandlers: OverviewViewModelRefreshHandler[] = []; private refreshState: 'InProgress' | 'Complete' = 'Complete'; constructor( @@ -62,21 +64,31 @@ export class OverviewViewModelActor { } } - private async updateFindings() { - const detectorInfo = new Map(); - this.overviewViewModel.detectors.forEach((detector) => { - detectorInfo.set(detector._id, { - logType: detector._source.detector_type, - name: detector._source.name, + private async updateFindings(signal: AbortSignal) { + const detectorInfo = new Map(); + this.overviewViewModel.detectors.forEach((detectorHit) => { + detectorInfo.set(detectorHit._id, { + logType: detectorHit._source.detector_type, + name: detectorHit._source.name, + detectorHit }); }); const detectorIds = detectorInfo.keys(); let findingItems: OverviewFindingItem[] = []; const ruleIds = new Set(); + const duration = getDuration({ + startTime: this.startTime, + endTime: this.endTime + }) try { for (let id of detectorIds) { - let detectorFindings: Finding[] = await DataStore.findings.getFindingsPerDetector(id); + let detectorFindings: Finding[] = await DataStore.findings.getFindingsPerDetector( + id, + detectorInfo.get(id)!.detectorHit, + signal, + duration + ); const logType = detectorInfo.get(id)?.logType; const detectorName = detectorInfo.get(id)?.name || ''; const detectorFindingItems: OverviewFindingItem[] = detectorFindings.map((finding) => { @@ -123,15 +135,21 @@ export class OverviewViewModelActor { this.overviewViewModel.findings = this.filterChartDataByTime(findingItems); } - private async updateAlerts() { + private async updateAlerts(signal: AbortSignal) { let alertItems: OverviewAlertItem[] = []; + const duration = getDuration({ + startTime: this.startTime, + endTime: this.endTime + }) try { for (let detector of this.overviewViewModel.detectors) { const id = detector._id; const detectorAlerts = await DataStore.alerts.getAlertsByDetector( id, - detector._source.name + detector._source.name, + signal, + duration ); const detectorAlertItems: OverviewAlertItem[] = detectorAlerts.map((alert) => ({ id: alert.id, @@ -154,14 +172,14 @@ export class OverviewViewModelActor { return this.overviewViewModel; } - public registerRefreshHandler(handler: OverviewViewModelRefreshHandler) { - this.refreshHandlers.push(handler); + public registerRefreshHandler(handler: OverviewViewModelRefreshHandler, allowPartialResults: boolean) { + allowPartialResults ? this.partialUpdateHandlers.push(handler) : this.fullUpdateHandlers.push(handler); } startTime = DEFAULT_DATE_RANGE.start; endTime = DEFAULT_DATE_RANGE.end; - public async onRefresh(startTime: string, endTime: string) { + public async onRefresh(startTime: string, endTime: string, signal: AbortSignal) { this.startTime = startTime; this.endTime = endTime; @@ -170,14 +188,23 @@ export class OverviewViewModelActor { } this.refreshState = 'InProgress'; - await this.updateDetectors(); - await this.updateFindings(); - await this.updateAlerts(); - this.refreshHandlers.forEach((handler) => { - handler(this.overviewViewModel); - }); + await this.runSteps([ + async () => { + await this.updateDetectors(); + this.updateResults(this.partialUpdateHandlers, false); + }, + async () => { + await this.updateFindings(signal); + this.updateResults(this.partialUpdateHandlers, false); + }, + async (signal: AbortSignal) => { + await this.updateAlerts(signal); + this.updateResults(this.partialUpdateHandlers, false); + } + ], signal); + this.updateResults(this.fullUpdateHandlers, true); this.refreshState = 'Complete'; } @@ -188,4 +215,24 @@ export class OverviewViewModelActor { return moment(dataItem.time).isBetween(moment(startMoment), moment(endMoment)); }); }; + + private updateResults(handlers: OverviewViewModelRefreshHandler[], modelLoadingComplete: boolean) { + handlers.forEach((handler) => { + handler(this.overviewViewModel, modelLoadingComplete); + }); + } + + private async runSteps(steps: Array<(signal: AbortSignal) => Promise>, signal: AbortSignal) { + for (let step of steps) { + if (signal.aborted) { + break; + } + + await step(signal); + + if (signal.aborted) { + break; + } + } + } } diff --git a/public/react-graph-vis.d.ts b/public/react-graph-vis.d.ts index 1b314c934..28e0227ed 100644 --- a/public/react-graph-vis.d.ts +++ b/public/react-graph-vis.d.ts @@ -4,10 +4,14 @@ */ declare module 'react-graph-vis' { - import { Network, NetworkEvents, Options, Node, Edge, DataSet, Data } from 'vis'; + import { Network as NetworkBase, NetworkEvents, Options, Node, Edge, DataSet, Data } from 'vis'; import { Component } from 'react'; - export { Network, NetworkEvents, Options, Node, Edge, DataSet, Data } from 'vis'; + export interface Network extends NetworkBase { + canvas: any; + } + + export { NetworkEvents, Options, Node, Edge, DataSet, Data } from 'vis'; export type GraphEvents = { [event in NetworkEvents]?: (params?: any) => void; diff --git a/public/services/AlertsService.ts b/public/services/AlertsService.ts index 3c27aa21f..8f74d9bfc 100644 --- a/public/services/AlertsService.ts +++ b/public/services/AlertsService.ts @@ -19,12 +19,14 @@ export default class AlertsService { getAlerts = async ( getAlertsParams: GetAlertsParams ): Promise> => { - const { detectorType, detector_id, size, sortOrder, startIndex } = getAlertsParams; + const { detectorType, detector_id, size, sortOrder, startIndex, startTime, endTime } = getAlertsParams; const baseQuery = { sortOrder: sortOrder || 'desc', size: size || 10000, startIndex: startIndex || 0, dataSourceId: dataSourceInfo.activeDataSource.id, + startTime, + endTime }; let query; diff --git a/public/services/FindingsService.ts b/public/services/FindingsService.ts index 306745008..4a5a890e4 100644 --- a/public/services/FindingsService.ts +++ b/public/services/FindingsService.ts @@ -17,15 +17,15 @@ export default class FindingsService { } getFindings = async ( - detectorParams: GetFindingsParams + getFindingsParams: GetFindingsParams ): Promise> => { - const findingIds = detectorParams.findingIds - ? JSON.stringify(detectorParams.findingIds) + const findingIds = getFindingsParams.findingIds + ? JSON.stringify(getFindingsParams.findingIds) : undefined; const query = { sortOrder: 'desc', size: 10000, - ...detectorParams, + ...getFindingsParams, findingIds, dataSourceId: dataSourceInfo.activeDataSource.id, }; diff --git a/public/store/AlertsStore.ts b/public/store/AlertsStore.ts index 52c326392..7d7489c30 100644 --- a/public/store/AlertsStore.ts +++ b/public/store/AlertsStore.ts @@ -6,6 +6,7 @@ import { NotificationsStart } from 'opensearch-dashboards/public'; import { AlertsService } from '../services'; import { errorNotificationToast } from '../utils/helpers'; +import { AlertResponse, Duration } from '../../types'; export class AlertsStore { constructor( @@ -13,20 +14,37 @@ export class AlertsStore { private readonly notifications: NotificationsStart ) {} - public async getAlertsByDetector(detectorId: string, detectorName: string) { + public async getAlertsByDetector( + detectorId: string, + detectorName: string, + signal: AbortSignal, + duration: Duration, + onPartialAlertsFetched?: (alerts: AlertResponse[]) => void + ) { let allAlerts: any[] = []; const maxAlertsReturned = 10000; let startIndex = 0; let alertsCount = 0; do { + if (signal.aborted) { + break; + } + const getAlertsRes = await this.service.getAlerts({ detector_id: detectorId, startIndex, size: maxAlertsReturned, + startTime: duration.startTime, + endTime: duration.endTime }); + if (signal.aborted) { + break; + } + if (getAlertsRes.ok) { + onPartialAlertsFetched?.(getAlertsRes.response.alerts) allAlerts = allAlerts.concat(getAlertsRes.response.alerts); alertsCount = getAlertsRes.response.alerts.length; } else { diff --git a/public/store/CorrelationsStore.ts b/public/store/CorrelationsStore.ts index e8e4ac34d..d142d0dd6 100644 --- a/public/store/CorrelationsStore.ts +++ b/public/store/CorrelationsStore.ts @@ -8,6 +8,7 @@ import { CorrelationFinding, CorrelationRule, CorrelationRuleQuery, + DetectorHit, ICorrelationsStore, IRulesStore, } from '../../types'; @@ -195,12 +196,32 @@ export class CorrelationsStore implements ICorrelationsStore { start_time, end_time ); - const allFindings = await this.fetchAllFindings(); - + const result: { finding1: CorrelationFinding; finding2: CorrelationFinding }[] = []; - + if (allCorrelationsRes.ok) { - allCorrelationsRes.response.findings.forEach(({ finding1, finding2 }) => { + const firstTenGrandCorrelations = allCorrelationsRes.response.findings.slice(0, 10000); + const allFindingIdsSet = new Set(); + firstTenGrandCorrelations.forEach(({ finding1, finding2 }) => { + allFindingIdsSet.add(finding1); + allFindingIdsSet.add(finding2); + }); + + const allFindingIds = Array.from(allFindingIdsSet); + let allFindings: { [id: string]: CorrelationFinding } = {}; + const maxFindingsFetchedInSingleCall = 10000; + + for (let i = 0; i < allFindingIds.length; i+= maxFindingsFetchedInSingleCall) { + const findingIds = allFindingIds.slice(i, i + maxFindingsFetchedInSingleCall); + const findings = await this.fetchAllFindings(findingIds); + allFindings = { + ...allFindings, + ...findings + } + } + + const maxNumberOfCorrelationsDisplayed = 10000; + allCorrelationsRes.response.findings.slice(0, maxNumberOfCorrelationsDisplayed).forEach(({ finding1, finding2 }) => { const f1 = allFindings[finding1]; const f2 = allFindings[finding2]; if (f1 && f2) @@ -222,55 +243,58 @@ export class CorrelationsStore implements ICorrelationsStore { public allFindings: { [id: string]: CorrelationFinding } = {}; - public async fetchAllFindings(): Promise<{ [id: string]: CorrelationFinding }> { + private async fetchAllFindings(findingIds: string[]): Promise<{ [id: string]: CorrelationFinding }> { const detectorsRes = await this.detectorsService.getDetectors(); const allRules = await this.rulesStore.getAllRules(); if (detectorsRes.ok) { - const detectors = detectorsRes.response.hits.hits; - let findings: { [id: string]: CorrelationFinding } = {}; - for (let detector of detectors) { - const detectorFindings = await DataStore.findings.getFindingsPerDetector(detector._id); - detectorFindings.forEach((f) => { - const rule = allRules.find((rule) => rule._id === f.queries[0].id); - findings[f.id] = { - ...f, - id: f.id, - logType: detector._source.detector_type, - detector: detector, - detectorName: detector._source.name, - timestamp: new Date(f.timestamp).toLocaleString(), - detectionRule: rule - ? { - name: rule._source.title, - severity: rule._source.level, - tags: rule._source.tags, - } - : { name: DEFAULT_EMPTY_DATA, severity: DEFAULT_EMPTY_DATA }, - }; - }); + const detectorsMap: { [id: string]: DetectorHit } = {}; + detectorsRes.response.hits.hits.forEach(detector => { + detectorsMap[detector._id] = detector; + }); + let findingsMap: { [id: string]: CorrelationFinding } = {}; + const findings = await DataStore.findings.getFindingsByIds(findingIds); + findings.forEach((f) => { + const detector = detectorsMap[f.detectorId]; + const rule = allRules.find((rule) => rule._id === f.queries[0].id); + findingsMap[f.id] = { + ...f, + id: f.id, + logType: detector._source.detector_type, + detector: detector, + detectorName: detector._source.name, + timestamp: new Date(f.timestamp).toLocaleString(), + detectionRule: rule + ? { + name: rule._source.title, + severity: rule._source.level, + tags: rule._source.tags, + } + : { name: DEFAULT_EMPTY_DATA, severity: DEFAULT_EMPTY_DATA }, + }; + }); - this.allFindings = findings; - } + this.allFindings = findingsMap; } return this.allFindings; } public async getCorrelatedFindings( - finding: string, + findingId: string, detector_type: string, nearby_findings = 20 ): Promise<{ finding: CorrelationFinding; correlatedFindings: CorrelationFinding[] }> { - const allFindings = await this.fetchAllFindings(); const response = await this.service.getCorrelatedFindings( - finding, + findingId, detector_type, nearby_findings ); if (response?.ok) { const correlatedFindings: CorrelationFinding[] = []; + const allFindingIds = response.response.findings.map(f => f.finding); + const allFindings = await this.fetchAllFindings(allFindingIds); response.response.findings.forEach((f) => { if (allFindings[f.finding]) { correlatedFindings.push({ @@ -282,15 +306,17 @@ export class CorrelationsStore implements ICorrelationsStore { }); return { - finding: allFindings[finding], + finding: allFindings[findingId], correlatedFindings, }; } + const finding = (await DataStore.findings.getFindingsByIds([findingId]))[0]; + return { finding: { - ...allFindings[finding], - id: finding, + ...finding, + id: findingId, logType: detector_type, timestamp: '', detectionRule: { name: '', severity: 'high' }, diff --git a/public/store/FindingsStore.ts b/public/store/FindingsStore.ts index 284a7b229..66e9995a5 100644 --- a/public/store/FindingsStore.ts +++ b/public/store/FindingsStore.ts @@ -8,9 +8,8 @@ import { DetectorsService, FindingsService } from '../services'; import { NotificationsStart } from 'opensearch-dashboards/public'; import { RouteComponentProps } from 'react-router-dom'; import { errorNotificationToast } from '../utils/helpers'; -import { FindingItemType } from '../pages/Findings/containers/Findings/Findings'; import { FindingDetailsFlyoutBaseProps } from '../pages/Findings/components/FindingDetailsFlyout'; -import { Finding, GetFindingsResponse, ServerResponse } from '../../types'; +import { DetectorHit, Duration, Finding, FindingItemType, GetFindingsResponse, ServerResponse } from '../../types'; export interface IFindingsStore { readonly service: FindingsService; @@ -23,9 +22,15 @@ export interface IFindingsStore { getFindingsByIds: (findingIds: string[]) => Promise; - getFindingsPerDetector: (detectorId: string) => Promise; + getFindingsPerDetector: ( + detectorId: string, + detector: DetectorHit, + signal: AbortSignal, + duration?: Duration, + onPartialFindingsFetched?: (findings: Finding[]) => void + ) => Promise; - getAllFindings: () => Promise; + getAllFindings: (signal: AbortSignal, duration?: { startTime: number; endTime: number; }, onPartialFindingsFetched?: (findings: Finding[]) => void) => Promise; setFlyoutCallback: ( flyoutCallback: (findingFlyout: FindingDetailsFlyoutBaseProps | null) => void @@ -111,25 +116,60 @@ export class FindingsStore implements IFindingsStore { return []; }; - public getFindingsPerDetector = async (detectorId: string): Promise => { - let allFindings: Finding[] = []; + public getFindingsPerDetector = async ( + detectorId: string, + detector: DetectorHit, + signal: AbortSignal, + duration?: Duration, + onPartialFindingsFetched?: (findings: FindingItemType[]) => void + ): Promise => { + let allFindings: FindingItemType[] = []; const findingsSize = 10000; - const firstGetFindingsRes = await this.service.getFindings({ + const getFindingsQueryParams = { detector_id: detectorId, startIndex: 0, size: findingsSize, - }); + startTime: duration?.startTime, + endTime: duration?.endTime + } + + if (signal.aborted) { + return allFindings; + } + + const firstGetFindingsRes = await this.service.getFindings(getFindingsQueryParams); if (firstGetFindingsRes.ok) { - allFindings = [...firstGetFindingsRes.response.findings]; + const extendedFindings = this.extendFindings(firstGetFindingsRes.response.findings, detector); + onPartialFindingsFetched?.(extendedFindings); + allFindings = [...extendedFindings]; let remainingFindings = firstGetFindingsRes.response.total_findings - findingsSize; let startIndex = findingsSize + 1; const getFindingsPromises: Promise>[] = []; while (remainingFindings > 0) { + + if (signal.aborted) { + return allFindings; + } + + const getFindingsPromise = this.service.getFindings({ + ...getFindingsQueryParams, + startIndex + }); + + if (signal.aborted) { + return allFindings; + } + getFindingsPromises.push( - this.service.getFindings({ detector_id: detectorId, startIndex, size: findingsSize }) + getFindingsPromise ); + getFindingsPromise.then((res): any => { + if (res.ok) { + onPartialFindingsFetched?.(this.extendFindings(res.response.findings, detector)); + } + }); remainingFindings -= findingsSize; startIndex += findingsSize; } @@ -138,7 +178,7 @@ export class FindingsStore implements IFindingsStore { findingsPromisesRes.forEach((response) => { if (response.status === 'fulfilled' && response.value.ok) { - allFindings = allFindings.concat(response.value.response.findings); + allFindings = allFindings.concat(this.extendFindings(response.value.response.findings, detector)); } }); } else { @@ -148,23 +188,19 @@ export class FindingsStore implements IFindingsStore { return allFindings; }; - public getAllFindings = async (): Promise => { + public getAllFindings = async ( + signal: AbortSignal, + duration?: Duration, + onPartialFindingsFetched?: (findings: FindingItemType[]) => void + ): Promise => { let allFindings: FindingItemType[] = []; const detectorsRes = await this.detectorsService.getDetectors(); if (detectorsRes.ok) { const detectors = detectorsRes.response.hits.hits; for (let detector of detectors) { - const findings = await this.getFindingsPerDetector(detector._id); - const findingsPerDetector: FindingItemType[] = findings.map((finding) => { - return { - ...finding, - detectorName: detector._source.name, - logType: detector._source.detector_type, - detector: detector, - correlations: [], - }; - }); + const findings = await this.getFindingsPerDetector(detector._id, detector, signal, duration, onPartialFindingsFetched); + const findingsPerDetector: FindingItemType[] = this.extendFindings(findings, detector); allFindings = allFindings.concat(findingsPerDetector); } } @@ -196,4 +232,16 @@ export class FindingsStore implements IFindingsStore { } as FindingDetailsFlyoutBaseProps; this.openFlyoutCallback(flyout); }; + + private extendFindings(findings: Finding[], detector: DetectorHit): FindingItemType[] { + return findings.map((finding) => { + return { + ...finding, + detectorName: detector._source.name, + logType: detector._source.detector_type, + detector: detector, + correlations: [], + }; + }); + } } diff --git a/public/utils/helpers.tsx b/public/utils/helpers.tsx index c5b579e33..f8566dfd0 100644 --- a/public/utils/helpers.tsx +++ b/public/utils/helpers.tsx @@ -39,11 +39,12 @@ import { IndexService, OpenSearchService } from '../services'; import { ruleSeverity, ruleTypes } from '../pages/Rules/utils/constants'; import { Handler } from 'vega-tooltip'; import _ from 'lodash'; -import { AlertCondition, LogType } from '../../types'; +import { AlertCondition, DateTimeFilter, Duration, LogType } from '../../types'; import { DataStore } from '../store/DataStore'; import { LogCategoryOptionView } from '../components/Utility/LogCategoryOption'; import { getLogTypeLabel } from '../pages/LogTypes/utils/helpers'; import { euiThemeVars } from '@osd/ui-shared-deps/theme'; +import dateMath from '@elastic/datemath'; export const parseStringsToOptions = (strings: string[]) => { return strings.map((str) => ({ id: str, label: str })); @@ -552,3 +553,13 @@ function getValueSetter(baseObject: any) { } }; } + +export function getDuration({ startTime, endTime }: DateTimeFilter): Duration { + const startMoment = dateMath.parse(startTime)!; + const endMoment = dateMath.parse(endTime)!; + + return { + startTime: startMoment.valueOf(), + endTime: endMoment.valueOf() + } +} diff --git a/server/routes/AlertRoutes.ts b/server/routes/AlertRoutes.ts index 743a4c2c1..0289801bf 100644 --- a/server/routes/AlertRoutes.ts +++ b/server/routes/AlertRoutes.ts @@ -22,6 +22,8 @@ export function setupAlertsRoutes(services: NodeServices, router: IRouter) { sortOrder: schema.maybe(schema.string()), size: schema.maybe(schema.number()), startIndex: schema.maybe(schema.number()), + startTime: schema.maybe(schema.number()), + endTime: schema.maybe(schema.number()) }), }, }, diff --git a/server/routes/FindingsRoutes.ts b/server/routes/FindingsRoutes.ts index 0bb659a4c..6302b1a26 100644 --- a/server/routes/FindingsRoutes.ts +++ b/server/routes/FindingsRoutes.ts @@ -26,6 +26,8 @@ export function setupFindingsRoutes(services: NodeServices, router: IRouter) { severity: schema.maybe(schema.string()), searchString: schema.maybe(schema.string()), findingIds: schema.maybe(schema.arrayOf(schema.string())), + startTime: schema.maybe(schema.number()), + endTime: schema.maybe(schema.number()) }), }, }, diff --git a/server/services/AlertService.ts b/server/services/AlertService.ts index 2eb44677c..36a52c4a3 100644 --- a/server/services/AlertService.ts +++ b/server/services/AlertService.ts @@ -30,10 +30,12 @@ export default class AlertService extends MDSEnabledClientService { response: OpenSearchDashboardsResponseFactory ): Promise | ResponseError>> => { try { - const { detectorType, detector_id, sortOrder, size } = request.query; + const { detectorType, detector_id, sortOrder, size, startTime, endTime } = request.query; const defaultParams = { sortOrder, size, + startTime, + endTime }; let params: GetAlertsParams; diff --git a/types/Alert.ts b/types/Alert.ts index 68f54d918..b85634f54 100644 --- a/types/Alert.ts +++ b/types/Alert.ts @@ -51,6 +51,8 @@ export type GetAlertsParams = { sortOrder?: string; size?: number; startIndex?: number; + startTime?: number; + endTime?: number; } & ( | { detector_id: string; diff --git a/types/Correlations.ts b/types/Correlations.ts index ca4ae61d2..1cf3cc222 100644 --- a/types/Correlations.ts +++ b/types/Correlations.ts @@ -137,7 +137,6 @@ export interface ICorrelationsStore { end_time: string ): Promise<{ finding1: CorrelationFinding; finding2: CorrelationFinding }[]>; allFindings: { [id: string]: CorrelationFinding }; - fetchAllFindings(): Promise<{ [id: string]: CorrelationFinding }>; } export type CorrelationLevelInfo = diff --git a/types/Overview.ts b/types/Overview.ts index 331f24d6f..a0035f4c2 100644 --- a/types/Overview.ts +++ b/types/Overview.ts @@ -21,7 +21,7 @@ export interface OverviewViewModel { alerts: OverviewAlertItem[]; } -export type OverviewViewModelRefreshHandler = (overviewState: OverviewViewModel) => void; +export type OverviewViewModelRefreshHandler = (overviewState: OverviewViewModel, modelUpdateComplete: boolean) => void; export interface OverviewProps extends RouteComponentProps, DataSourceProps { getStartedDismissedOnce: boolean; diff --git a/types/index.ts b/types/index.ts index 958772eb0..8b0b8c7e6 100644 --- a/types/index.ts +++ b/types/index.ts @@ -19,3 +19,4 @@ export * from './Metrics'; export * from './SecurityAnalyticsContext'; export * from './DataSourceContext'; export * from './DataSource'; +export * from './shared'; diff --git a/types/shared.ts b/types/shared.ts new file mode 100644 index 000000000..ba22d93ff --- /dev/null +++ b/types/shared.ts @@ -0,0 +1,27 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +*/ + +import { CorrelationFinding } from "./Correlations"; +import { DetectorHit } from "./Detector"; +import { Finding } from "./Finding"; + +export interface Duration { + startTime: number; + endTime: number; +} + +export type FindingItemType = Finding & { + logType: string; + detectorName: string; + detector: DetectorHit; + correlations: CorrelationFinding[]; +}; + +export interface FindingDetectorMetadata { + detectorName: string; + logType: string; + detector: DetectorHit + correlations: [] +} \ No newline at end of file