From bdd3f002bbef3a8cabf18e7e26aad68adee9ed4b Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Mon, 16 Dec 2024 22:07:00 -0800 Subject: [PATCH 1/5] De-dupe Alerts generated by Aggregation Sigma Rules fix Signed-off-by: Riya Saxena --- .../monitor/TransportIndexThreatIntelMonitorAction.java | 3 ++- .../transport/TransportIndexDetectorAction.java | 8 +++++--- .../securityanalytics/alerts/AlertingServiceTests.java | 6 ++++-- .../threatIntel/model/monitor/ThreatIntelInputTests.java | 3 ++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index 4316e4711..bc0875a13 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -241,7 +241,8 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) new DataSources(), false, null, - PLUGIN_OWNER_FIELD + PLUGIN_OWNER_FIELD, + true ); } catch (Exception e) { String error = "Error occurred while parsing monitor."; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 8f4c4f1fd..7c7dcd8dc 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -471,6 +471,8 @@ public void onResponse(Map> ruleFieldMappings) { @Override public void onResponse(Collection indexMonitorRequests) { if (detector.getRuleIdMonitorIdMap().containsKey(CHAINED_FINDINGS_MONITOR_STRING)) { + // set the toggle flag disable + String cmfId = detector.getRuleIdMonitorIdMap().get(CHAINED_FINDINGS_MONITOR_STRING); if (shouldAddChainedFindingDocMonitor(indexMonitorRequests.isEmpty(), rulesById)) { monitorsToBeUpdated.add(createDocLevelMonitorMatchAllRequest(detector, RefreshPolicy.IMMEDIATE, cmfId, Method.PUT, rulesById)); @@ -797,7 +799,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), enableDetectorWithDedicatedQueryIndices, null, PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, null, PLUGIN_OWNER_FIELD, true); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -902,7 +904,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), enableDetectorWithDedicatedQueryIndices, true, PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, true, PLUGIN_OWNER_FIELD, false); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1078,7 +1080,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), false, null, PLUGIN_OWNER_FIELD); + true), false, null, PLUGIN_OWNER_FIELD, true); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 82d6ecc5c..84f70830c 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -97,7 +97,8 @@ public void testGetAlerts_success() { new DataSources(), true, null, - TransportIndexDetectorAction.PLUGIN_OWNER_FIELD + TransportIndexDetectorAction.PLUGIN_OWNER_FIELD, + true ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), List.of("finding_id_1"), @@ -133,7 +134,8 @@ public void testGetAlerts_success() { new DataSources(), true, null, - TransportIndexDetectorAction.PLUGIN_OWNER_FIELD + TransportIndexDetectorAction.PLUGIN_OWNER_FIELD, + true ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), List.of("finding_id_1"), diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index 462873959..3135a2524 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -59,7 +59,8 @@ public void testThreatInputSerde() throws IOException { new DataSources(), false, null, - "security_analytics" + "security_analytics", + true ); BytesStreamOutput monitorOut = new BytesStreamOutput(); monitor.writeTo(monitorOut); From aeda87763a79b3e6699e11e3f0796ab5a5d515b2 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Mon, 16 Dec 2024 22:13:17 -0800 Subject: [PATCH 2/5] De-dupe Alerts generated by Aggregation Sigma Rules fix Signed-off-by: Riya Saxena --- .../transport/TransportIndexDetectorAction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 7c7dcd8dc..2a77a90b6 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -471,8 +471,6 @@ public void onResponse(Map> ruleFieldMappings) { @Override public void onResponse(Collection indexMonitorRequests) { if (detector.getRuleIdMonitorIdMap().containsKey(CHAINED_FINDINGS_MONITOR_STRING)) { - // set the toggle flag disable - String cmfId = detector.getRuleIdMonitorIdMap().get(CHAINED_FINDINGS_MONITOR_STRING); if (shouldAddChainedFindingDocMonitor(indexMonitorRequests.isEmpty(), rulesById)) { monitorsToBeUpdated.add(createDocLevelMonitorMatchAllRequest(detector, RefreshPolicy.IMMEDIATE, cmfId, Method.PUT, rulesById)); From 609ab3894ffae661d4109992e48cc310d0758893 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Tue, 17 Dec 2024 11:29:52 -0800 Subject: [PATCH 3/5] De-dupe Alerts generated by Aggregation Sigma Rules fix Signed-off-by: Riya Saxena --- .../TransportIndexThreatIntelMonitorAction.java | 6 +++--- .../transport/TransportIndexDetectorAction.java | 10 +++++----- .../securityanalytics/alerts/AlertingServiceTests.java | 6 ++---- .../model/monitor/ThreatIntelInputTests.java | 6 +++--- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index bc0875a13..c4902b99b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -202,7 +202,8 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput( String.format("threat intel input for monitor named %s", request.getMonitor().getName()), request.getMonitor().getIndices(), - Collections.emptyList() // no percolate queries + Collections.emptyList(), // no percolate queries + true ); List perIocTypeScanInputs = request.getMonitor().getPerIocTypeScanInputList().stream().map( it -> new PerIocTypeScanInput(it.getIocType(), it.getIndexToFieldsMap()) @@ -241,8 +242,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) new DataSources(), false, null, - PLUGIN_OWNER_FIELD, - true + PLUGIN_OWNER_FIELD ); } catch (Exception e) { String error = "Error occurred while parsing monitor."; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 2a77a90b6..75cad056a 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -772,7 +772,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List docLevelQueries.add(docLevelQuery); } docLevelQueries.addAll(threatIntelQueries); - DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); + DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries, true); docLevelMonitorInputs.add(docLevelMonitorInput); List triggers = new ArrayList<>(); @@ -797,7 +797,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), enableDetectorWithDedicatedQueryIndices, null, PLUGIN_OWNER_FIELD, true); + true), enableDetectorWithDedicatedQueryIndices, null, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -877,7 +877,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( ); docLevelQueries.add(docLevelQuery); - DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries); + DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), detector.getInputs().get(0).getIndices(), docLevelQueries, false); docLevelMonitorInputs.add(docLevelMonitorInput); List triggers = new ArrayList<>(); @@ -902,7 +902,7 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), enableDetectorWithDedicatedQueryIndices, true, PLUGIN_OWNER_FIELD, false); + true), enableDetectorWithDedicatedQueryIndices, true, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1078,7 +1078,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), false, null, PLUGIN_OWNER_FIELD, true); + true), false, null, PLUGIN_OWNER_FIELD); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 84f70830c..82d6ecc5c 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -97,8 +97,7 @@ public void testGetAlerts_success() { new DataSources(), true, null, - TransportIndexDetectorAction.PLUGIN_OWNER_FIELD, - true + TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), List.of("finding_id_1"), @@ -134,8 +133,7 @@ public void testGetAlerts_success() { new DataSources(), true, null, - TransportIndexDetectorAction.PLUGIN_OWNER_FIELD, - true + TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), List.of("finding_id_1"), diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index 3135a2524..d56969de0 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -50,7 +50,8 @@ public void testThreatInputSerde() throws IOException { bytes, new DocLevelMonitorInput("threat intel input", List.of("index1", "index2"), - emptyList() + emptyList(), + true ) ) ), @@ -59,8 +60,7 @@ public void testThreatInputSerde() throws IOException { new DataSources(), false, null, - "security_analytics", - true + "security_analytics" ); BytesStreamOutput monitorOut = new BytesStreamOutput(); monitor.writeTo(monitorOut); From 42f19462daf7720c0bd5d29774823d2cd2fb8768 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Tue, 17 Dec 2024 13:51:15 -0800 Subject: [PATCH 4/5] tests fix Signed-off-by: Riya Saxena --- .../java/org/opensearch/securityanalytics/alerts/AlertsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index fe97a13be..4b8f983e3 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -956,7 +956,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null); getAlertsBody = asMap(getAlertsResponse); // TODO enable asserts here when able - Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); } @Ignore From 14d7d9e3b2cda1dfd515f51d0fb2f4c396825e63 Mon Sep 17 00:00:00 2001 From: Riya Saxena Date: Wed, 18 Dec 2024 07:50:17 -0800 Subject: [PATCH 5/5] tests fix Signed-off-by: Riya Saxena --- .../java/org/opensearch/securityanalytics/alerts/AlertsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 4b8f983e3..fe97a13be 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -956,7 +956,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params1, null); getAlertsBody = asMap(getAlertsResponse); // TODO enable asserts here when able - Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); } @Ignore