Skip to content

Commit

Permalink
Feature findings api enhancements (opensearch-project#914)
Browse files Browse the repository at this point in the history 
* get all findings as part of findings API enhancement

Signed-off-by: Riya Saxena <[email protected]>

* findingsAPI feature enhancements (address comments to prev PR)

Signed-off-by: Riya Saxena <[email protected]>

* findingsAPI feature enhancements (address comments to prev PR)

Signed-off-by: Riya Saxena <[email protected]>

* added support for  param in Finding API

Signed-off-by: Riya Saxena <[email protected]>

* added detectionType as param for Findings API enhancements

Signed-off-by: Riya Saxena <[email protected]>

* added few tests to validate findings by params

Signed-off-by: Riya Saxena <[email protected]>

* added test for searchString param in FindingsAPI

Signed-off-by: Riya Saxena <[email protected]>

* adding addiional params findingIds, startTime and endTime as findings API enhancement

Signed-off-by: Riya Saxena <[email protected]>

* added params in getFindingsByDetectorId func

* changed the startTime and endTime req input format

* fix merge conflixt

* fix integ test failures in findings API

* fix integ tests

* refactored the logic

Signed-off-by: Riya Saxena <[email protected]>

* remove unused imports

* address the pr comments

Signed-off-by: Riya Saxena <[email protected]>

* address pr comments

Signed-off-by: Riya Saxena <[email protected]>

* SA integ tests fix

* SA integ tests fix

* fix integ tests for findings

Signed-off-by: Subhobrata Dey <[email protected]>

* fix conflixt errors

Signed-off-by: Riya Saxena <[email protected]>

* fix conflixt errors

Signed-off-by: Riya Saxena <[email protected]>

* fix conflixt errors

Signed-off-by: Riya Saxena <[email protected]>

* fix conflixt errors

Signed-off-by: Riya Saxena <[email protected]>

* fix integ tests

Signed-off-by: Riya Saxena <[email protected]>

* fix integ tests

Signed-off-by: Riya Saxena <[email protected]>

* fix integ tests

Signed-off-by: Riya Saxena <[email protected]>

* fix flaky integ tests

Signed-off-by: Riya Saxena <[email protected]>

* address pr comments

Signed-off-by: Riya Saxena <[email protected]>

---------

Signed-off-by: Riya Saxena <[email protected]>
Signed-off-by: Riya <[email protected]>
Signed-off-by: Subhobrata Dey <[email protected]>
Co-authored-by: Subhobrata Dey <[email protected]>
(cherry picked from commit 9b59f61)
  • Loading branch information
@riysaxen-amzn
riysaxen-amzn committed Mar 14, 2024
1 parent fbccff4 commit 810bb40
Show file tree
Hide file tree
Showing 6 changed files with 728 additions and 69 deletions.
63 changes: 61 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.join.ScoreMode;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.core.action.ActionListener;
import org.opensearch.client.Client;
Expand All @@ -21,6 +22,11 @@
import org.opensearch.commons.alerting.model.FindingWithDocs;
import org.opensearch.commons.alerting.model.Table;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.PrefixQueryBuilder;
import org.opensearch.index.query.NestedQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.securityanalytics.action.FindingDto;
import org.opensearch.securityanalytics.action.GetDetectorAction;
import org.opensearch.securityanalytics.action.GetDetectorRequest;
Expand Down Expand Up @@ -129,13 +135,13 @@ public void getFindingsByMonitorIds(
ActionListener<GetFindingsResponse> listener
) {

BoolQueryBuilder queryBuilder = getBoolQueryBuilder(detectionType, severity, findingIds, startTime, endTime);
org.opensearch.commons.alerting.action.GetFindingsRequest req =
new org.opensearch.commons.alerting.action.GetFindingsRequest(
null,
table,
null,
findingIndexName,
monitorIds
findingIndexName, monitorIds, queryBuilder
);

AlertingPluginInterface.INSTANCE.getFindings((NodeClient) client, req, new ActionListener<>() {
Expand Down Expand Up @@ -163,6 +169,59 @@ public void onFailure(Exception e) {

}

private static BoolQueryBuilder getBoolQueryBuilder(String detectionType, String severity, List<String> findingIds, Instant startTime, Instant endTime) {
// Construct the query within the search source builder
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

if (detectionType != null && !detectionType.isBlank()) {
QueryBuilder nestedQuery;
if (detectionType.equalsIgnoreCase("threat")) {
nestedQuery = QueryBuilders.boolQuery().filter(
new PrefixQueryBuilder("queries.id", "threat_intel_")
);
} else {
nestedQuery = QueryBuilders.boolQuery().mustNot(
new PrefixQueryBuilder("queries.id", "threat_intel_")
);
}

// Create a nested query builder
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(
"queries",
nestedQuery,
ScoreMode.None
);

// Add the nested query to the bool query
boolQueryBuilder.must(nestedQueryBuilder);
}

if (findingIds != null && !findingIds.isEmpty()) {
boolQueryBuilder.filter(QueryBuilders.termsQuery("id", findingIds));
}


if (startTime != null && endTime != null) {
long startTimeMillis = startTime.toEpochMilli();
long endTimeMillis = endTime.toEpochMilli();
QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp")
.from(startTimeMillis) // Greater than or equal to start time
.to(endTimeMillis); // Less than or equal to end time
boolQueryBuilder.filter(timeRangeQuery);
}

if (severity != null) {
boolQueryBuilder.must(QueryBuilders.nestedQuery(
"queries",
QueryBuilders.boolQuery().should(
QueryBuilders.matchQuery("queries.tags", severity)
),
ScoreMode.None
));
}
return boolQueryBuilder;
}

void setIndicesAdminClient(Client client) {
this.client = client;
}
Expand Down
130 changes: 77 additions & 53 deletions src/main/java/org/opensearch/securityanalytics/transport/TransportGetFindingsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import java.io.IOException;
import java.util.List;
import java.util.Locale;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.search.join.ScoreMode;
Expand All @@ -23,6 +23,7 @@
import org.opensearch.common.settings.Settings;
import org.opensearch.commons.authuser.User;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.index.query.MatchAllQueryBuilder;
import org.opensearch.index.query.NestedQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.core.rest.RestStatus;
Expand All @@ -41,12 +42,12 @@
import org.opensearch.tasks.Task;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;


import static org.opensearch.securityanalytics.util.DetectorUtils.DETECTOR_TYPE_PATH;
import static org.opensearch.securityanalytics.util.DetectorUtils.MAX_DETECTORS_SEARCH_SIZE;
import static org.opensearch.securityanalytics.util.DetectorUtils.NO_DETECTORS_FOUND;
import static org.opensearch.securityanalytics.util.DetectorUtils.NO_DETECTORS_FOUND_FOR_PROVIDED_TYPE;

public class TransportGetFindingsAction extends HandledTransportAction<GetFindingsRequest, GetFindingsResponse> implements SecureTransportAction {

private final TransportSearchDetectorAction transportSearchDetectorAction;

private final NamedXContentRegistry xContentRegistry;
Expand Down Expand Up @@ -103,70 +104,93 @@ protected void doExecute(Task task, GetFindingsRequest request, ActionListener<G
actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN));
return;
}

if (request.getLogType() == null) {
if (request.getDetectorId() != null) {
// Get the Findings by DetectorId
findingsService.getFindingsByDetectorId(
request.getDetectorId(),
request.getTable(),
request.getSeverity(),
request.getDetectionType(),
request.getFindingIds(),
request.getStartTime(),
request.getEndTime(),
actionListener
);
} else {
// "detector" is nested type, so we have to use nested query
NestedQueryBuilder queryBuilder =
QueryBuilders.nestedQuery(
"detector",
QueryBuilders.boolQuery().must(
QueryBuilders.matchQuery(
DETECTOR_TYPE_PATH,
request.getLogType()
// Get the Findings when logType is not null
SearchRequest searchRequest = getSearchDetectorsRequest(request);
getFindingsFromDetectors(request, actionListener, searchRequest);
}
}

private void getFindingsFromDetectors(GetFindingsRequest findingsRequest, ActionListener<GetFindingsResponse> findingsResponseActionListener, SearchRequest searchRequest) {
transportSearchDetectorAction.execute(new SearchDetectorRequest(searchRequest), new ActionListener<>() {
@Override
public void onResponse(SearchResponse searchResponse) {
try {
List<Detector> detectors = DetectorUtils.getDetectors(searchResponse, xContentRegistry);
if (detectors.size() == 0) {
findingsResponseActionListener.onFailure(
SecurityAnalyticsException.wrap(
new OpenSearchStatusException(
findingsRequest.getLogType() == null ? NO_DETECTORS_FOUND : NO_DETECTORS_FOUND_FOR_PROVIDED_TYPE, RestStatus.NOT_FOUND
)
)
),
ScoreMode.None
);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.fetchSource(true);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(Detector.DETECTORS_INDEX);
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());

transportSearchDetectorAction.execute(new SearchDetectorRequest(searchRequest), new ActionListener<>() {
@Override
public void onResponse(SearchResponse searchResponse) {
try {
List<Detector> detectors = DetectorUtils.getDetectors(searchResponse, xContentRegistry);
if (detectors.size() == 0) {
actionListener.onFailure(
SecurityAnalyticsException.wrap(
new OpenSearchStatusException(
"No detectors found for provided type", RestStatus.NOT_FOUND
)
)
);
return;
}
findingsService.getFindings(
detectors,
request.getLogType(),
request.getTable(),
actionListener
);
} catch (IOException e) {
actionListener.onFailure(e);
return;
}
findingsService.getFindings(
detectors,
findingsRequest.getLogType() == null ? "*" : findingsRequest.getLogType(),
findingsRequest.getTable(),
findingsRequest.getSeverity(),
findingsRequest.getDetectionType(),
findingsRequest.getFindingIds(),
findingsRequest.getStartTime(),
findingsRequest.getEndTime(),
findingsResponseActionListener
);
} catch (IOException e) {
findingsResponseActionListener.onFailure(e);
}
}
@Override
public void onFailure(Exception e) {
findingsResponseActionListener.onFailure(e);
}
});
}

@Override
public void onFailure(Exception e) {
actionListener.onFailure(e);
}
});
private static SearchRequest getSearchDetectorsRequest(GetFindingsRequest findingsRequest) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (findingsRequest.getLogType() != null) {
NestedQueryBuilder queryBuilder = QueryBuilders.nestedQuery(
"detector",
QueryBuilders.boolQuery().must(
QueryBuilders.matchQuery(
DETECTOR_TYPE_PATH,
findingsRequest.getLogType()
)
),
ScoreMode.None
);
searchSourceBuilder.query(queryBuilder);
}
else {
MatchAllQueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
searchSourceBuilder.query(queryBuilder);
}
searchSourceBuilder.size(MAX_DETECTORS_SEARCH_SIZE); // Set the size to 10000
searchSourceBuilder.fetchSource(true);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(Detector.DETECTORS_INDEX);
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());
return searchRequest;
}

private void setFilterByEnabled(boolean filterByEnabled) {
this.filterByEnabled = filterByEnabled;
}

}
}
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ public class DetectorUtils {

public static final String DETECTOR_TYPE_PATH = "detector.detector_type";
public static final String DETECTOR_ID_FIELD = "detector_id";
public static final String NO_DETECTORS_FOUND = "No detectors found ";
public static final String NO_DETECTORS_FOUND_FOR_PROVIDED_TYPE = "No detectors found for provided type";
public static final int MAX_DETECTORS_SEARCH_SIZE = 10000;

public static SearchResponse getEmptySearchResponse() {
return new SearchResponse(new InternalSearchResponse(
Expand Down Expand Up @@ -132,4 +135,4 @@ private static boolean checkIfRuleIsAggAndTriggerable(Rule rule, Set<String> rul
}


}
}
Loading

0 comments on commit 810bb40

Please sign in to comment.