From 865ad34bb6f5e723018f4232d4d8a72da4779a34 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Sun, 10 Mar 2024 17:56:14 -0700 Subject: [PATCH] Fix get mappings view API incorrectly returning ecs path (#867) * add logic and integ tests to not add duplicate to log-types-config index Signed-off-by: Joanne Wang * change naming for existingFieldMapping and change contains to equals Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../logtype/LogTypeService.java | 23 ++++-- .../securityanalytics/TestHelpers.java | 82 +++++++++++++++++-- .../resthandler/OCSFDetectorRestApiIT.java | 46 +++++++++++ 3 files changed, 139 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 85f2242ad..2a0665b23 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -348,15 +348,26 @@ private List mergeFieldMappings(List existingF List newFieldMappings = new ArrayList<>(); fieldMappingDocs.forEach( newFieldMapping -> { Optional foundFieldMappingDoc = Optional.empty(); - for (FieldMappingDoc e: existingFieldMappings) { - if (e.getRawField().equals(newFieldMapping.getRawField())) { + for (FieldMappingDoc existingFieldMapping: existingFieldMappings) { + if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) { if (( - e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && - e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) + existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && + existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) ) || ( - e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null + existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null )) { - foundFieldMappingDoc = Optional.of(e); + foundFieldMappingDoc = Optional.of(existingFieldMapping); + } + // Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings + // Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed + } else { + String id = existingFieldMapping.getId(); + int indexOfPipe = id.indexOf("|"); + if (indexOfPipe != -1) { + String ecsIdField = id.substring(indexOfPipe + 1); + if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) { + foundFieldMappingDoc = Optional.of(existingFieldMapping); + } } } } diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 7f4439ce9..a951415ee 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -6,19 +6,19 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.tests.util.LuceneTestCase; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.model.action.Throttle; import org.opensearch.commons.authuser.User; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.CorrelationQuery; @@ -239,6 +239,35 @@ public static String randomRule() { "level: high"; } + public static String randomRuleWithRawField() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " eventName: testinghere\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomNullRule() { return "title: null field\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -1774,6 +1803,46 @@ public static String randomDoc() { "}"; } + public static String randomNetworkDoc() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":9532,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"id.orig_h\": \"123.12.123.12\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } + public static String randomCloudtrailAggrDoc(String eventType, String accountId) { return "{\n" + " \"AccountName\": \"" + accountId + "\",\n" + @@ -1791,6 +1860,7 @@ public static String randomVpcFlowDoc() { " \"srcport\": 9000,\n" + " \"dstport\": 8000,\n" + " \"severity_id\": \"-1\",\n" + + " \"id.orig_h\": \"1.2.3.4\",\n" + " \"class_name\": \"Network Activity\"\n" + "}"; } @@ -2366,7 +2436,7 @@ public static List randomLowerCaseStringList() { stringList.add(randomLowerCaseString()); return stringList; } - + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java index 07ab7164d..8c1a14b52 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java @@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException { assertEquals(24, unmappedFieldAliases.size()); } + @SuppressWarnings("unchecked") + public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException { + String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings()); + + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "cloudtrail"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + Map props = (Map) respMap.get("properties"); + Assert.assertEquals(18, props.size()); + // Verify unmapped index fields + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + assertEquals(20, unmappedIndexFields.size()); + // Verify unmapped field aliases + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + assertEquals(25, unmappedFieldAliases.size()); + + // create a cloudtrail rule with a raw field + String rule = randomRuleWithRawField(); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + // check the mapping view API again to ensure it's the same after rule is created + Response response2 = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode()); + Map respMap2 = responseAsMap(response2); + // Verify alias mappings + Map props2 = (Map) respMap2.get("properties"); + Assert.assertEquals(18, props2.size()); + // Verify unmapped index fields + List unmappedIndexFields2 = (List) respMap2.get("unmapped_index_fields"); + assertEquals(20, unmappedIndexFields2.size()); + // Verify unmapped field aliases + List unmappedFieldAliases2 = (List) respMap2.get("unmapped_field_aliases"); + assertEquals(25, unmappedFieldAliases2.size()); + // Verify that first response and second response are the same after rule was indexed + assertEquals(props, props2); + assertEquals(unmappedIndexFields, unmappedIndexFields2); + assertEquals(unmappedFieldAliases, unmappedFieldAliases2); + } + @SuppressWarnings("unchecked") public void testOCSFVpcflowGetMappingsViewApi() throws IOException { String index = createTestIndex("vpcflow", ocsfVpcflowMappings());