From c9ba86e97b6e5cb4823c8cda47527e8c9c53df8b Mon Sep 17 00:00:00 2001 From: Dennis Toepker Date: Mon, 23 Dec 2024 11:55:04 -0800 Subject: [PATCH] OCSF1.1 Fixes Signed-off-by: Dennis Toepker --- .../mapper/MapperService.java | 18 +- src/main/resources/OSMapping/waf_logtype.json | 4 +- .../resthandler/OCSFDetectorRestApiIT.java | 156 +++++++++++++++++- 3 files changed, 164 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 4ea64d1ef..b48f00cbb 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -232,10 +232,10 @@ public void onResponse(List mappings) { for (LogType.Mapping mapping : mappings) { if (indexFields.contains(mapping.getRawField())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField())); - } else if (indexFields.contains(mapping.getOcsf())) { - aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } else if (indexFields.contains(mapping.getOcsf11())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11())); + } else if (indexFields.contains(mapping.getOcsf())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } } aliasMappingsObj.field("properties", aliasMappingFields); @@ -484,8 +484,8 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { for (LogType.Mapping requiredField : requiredFields) { String alias = requiredField.getEcs(); String rawPath = requiredField.getRawField(); - String ocsfPath = requiredField.getOcsf(); String ocsf11Path = requiredField.getOcsf11(); + String ocsfPath = requiredField.getOcsf(); if (allFieldsFromIndex.contains(rawPath)) { // if the alias was already added into applyable aliases, then skip to avoid duplicates if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) { @@ -497,12 +497,12 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } pathsOfApplyableAliases.add(rawPath); } - } else if (allFieldsFromIndex.contains(ocsfPath)) { - applyableAliases.add(alias); - pathsOfApplyableAliases.add(ocsfPath); } else if (allFieldsFromIndex.contains(ocsf11Path)) { applyableAliases.add(alias); pathsOfApplyableAliases.add(ocsf11Path); + } else if (allFieldsFromIndex.contains(ocsfPath)) { + applyableAliases.add(alias); + pathsOfApplyableAliases.add(ocsfPath); } else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) { if (alias != null) { // we don't want to send back aliases which have same name as existing field in index @@ -524,10 +524,10 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { Map> aliasMappingFields = new HashMap<>(); XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject(); for (LogType.Mapping mapping : requiredFields) { - if (allFieldsFromIndex.contains(mapping.getOcsf())) { - aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); - } else if (allFieldsFromIndex.contains(mapping.getOcsf11())) { + if (allFieldsFromIndex.contains(mapping.getOcsf11())) { aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11())); + } else if (allFieldsFromIndex.contains(mapping.getOcsf())) { + aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf())); } else if (mapping.getEcs() != null) { shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases); } else if (mapping.getEcs() == null) { diff --git a/src/main/resources/OSMapping/waf_logtype.json b/src/main/resources/OSMapping/waf_logtype.json index 3433b56c1..352a5e155 100644 --- a/src/main/resources/OSMapping/waf_logtype.json +++ b/src/main/resources/OSMapping/waf_logtype.json @@ -57,12 +57,12 @@ { "raw_field":"httpRequest.headers.value", "ecs":"waf.request.headers.value", - "ocsf": "http_request.http_headers[].value" + "ocsf": "http_request.http_headers.value" }, { "raw_field":"httpRequest.headers.name", "ecs":"waf.request.headers.name", - "ocsf": "http_request.http_headers[].name" + "ocsf": "http_request.http_headers.name" } ] } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index c457c0e1e..93efe6287 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -10,6 +10,7 @@ import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; @@ -17,9 +18,11 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -433,7 +436,7 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { Assert.assertEquals(18, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields.size()); + assertEquals(21, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases.size()); @@ -455,7 +458,8 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept Assert.assertEquals(18, props.size()); // Verify unmapped index fields List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields.size()); + + assertEquals(21, unmappedIndexFields.size()); // Verify unmapped field aliases List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases.size()); @@ -475,7 +479,7 @@ public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOExcept Assert.assertEquals(18, props2.size()); // Verify unmapped index fields List unmappedIndexFields2 = (List) respMap2.get("unmapped_index_fields"); - assertEquals(20, unmappedIndexFields2.size()); + assertEquals(21, unmappedIndexFields2.size()); // Verify unmapped field aliases List unmappedFieldAliases2 = (List) respMap2.get("unmapped_field_aliases"); assertEquals(24, unmappedFieldAliases2.size()); @@ -595,6 +599,93 @@ public void testRawRoute53GetMappingsViewApi() throws IOException { assertEquals(8, unmappedFieldAliases.size()); } + public void testCloudtrailPrincipalIdAndArnFieldsGenerateFinding() throws IOException { + // create an index with OCSF1.1 fields actor.user.uid and actor.user.uid_alt + String indexName = "test_index"; + String index = createTestIndex(indexName, ocsf11ReducedCloudtrailMappings()); + + // create the cloudtrail mappings + createMappingsAPI(indexName, "cloudtrail"); + + // create the custom rule + String rule = ocsf11Rule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String ruleId = responseBody.get("_id").toString(); + + // create the detector that uses only the custom rule + Detector detector = randomDetector( + "cloudtrail-detector", + "cloudtrail", + null, + List.of( + new DetectorInput( + "cloudtrail detector for security analytics", + List.of(indexName), + List.of(new DetectorRule(ruleId)), + List.of() + ) + ), + List.of( + new DetectorTrigger( + null, + "cloudtrail-trigger", + "1", + List.of("cloudtrail"), + List.of(ruleId), + List.of(), + List.of(), + List.of(), + List.of() + ) + ), + null, + true, + null, + null, + false + ); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + // get the underlying alerting monitor for the detector so we can manually execute it + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", "cloudtrail", detectorType.toLowerCase(Locale.ROOT)); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + // index a document that should trigger a finding + indexDoc(index, "1", ocsf11Doc()); + + // execute detector by executing its underlying monitor + executeAlertingMonitor(monitorId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + } + private String rawCloudtrailDoc() { return "{\n" + " \"eventVersion\": \"1.03\",\n" + @@ -2810,4 +2901,63 @@ private String rawVpcFlowMappings() { " }\n" + " }"; } + + private String ocsf11ReducedCloudtrailMappings() { + return "\"properties\": {\n" + + " \"actor.user.uid_alt\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " },\n" + + " \"actor.user.uid\": {\n" + + " \"type\": \"text\",\n" + + " \"fields\": {\n" + + " \"keyword\": {\n" + + " \"type\": \"keyword\",\n" + + " \"ignore_above\": 256\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + + private String ocsf11Rule() { + return "title: Cloudtrail Principal ID Rule\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a123\n" + + "description: A rule that checks specifically for the cloudtrail principal ID field\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " aws.cloudtrail.user_identity.principalId: abc\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + + public String ocsf11Doc() { + return "{\n" + + "\"actor.user.uid_alt\":\"abc\",\n" + + "\"actor.user.uid\":\"def\"\n" + + "}"; + } } \ No newline at end of file