Skip to content

Commit

Permalink
Fix get mappings view API incorrectly returning ecs path (#867)
Browse files Browse the repository at this point in the history 
* add logic and integ tests to not add duplicate to log-types-config index

Signed-off-by: Joanne Wang <[email protected]>

* change naming for existingFieldMapping and change contains to equals

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon
jowg-amazon committed Mar 15, 2024
1 parent 99931b4 commit 865ad34
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 12 deletions.
23 changes: 17 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,15 +348,26 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( newFieldMapping -> {
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
for (FieldMappingDoc e: existingFieldMappings) {
if (e.getRawField().equals(newFieldMapping.getRawField())) {
for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
if ((
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
)) {
foundFieldMappingDoc = Optional.of(e);
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
// Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
// Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
} else {
String id = existingFieldMapping.getId();
int indexOfPipe = id.indexOf("|");
if (indexOfPipe != -1) {
String ecsIdField = id.substring(indexOfPipe + 1);
if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
foundFieldMappingDoc = Optional.of(existingFieldMapping);
}
}
}
}
Expand Down
82 changes: 76 additions & 6 deletions src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,19 @@

import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.commons.alerting.model.IntervalSchedule;
import org.opensearch.commons.alerting.model.Schedule;
import org.opensearch.commons.alerting.model.action.Action;
import org.opensearch.commons.alerting.model.action.Throttle;
import org.opensearch.commons.authuser.User;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.script.Script;
import org.opensearch.script.ScriptType;
import org.opensearch.securityanalytics.model.CorrelationQuery;
Expand Down Expand Up @@ -239,6 +239,35 @@ public static String randomRule() {
"level: high";
}

public static String randomRuleWithRawField() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" eventName: testinghere\n" +
" condition: selection\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomNullRule() {
return "title: null field\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
Expand Down Expand Up @@ -1774,6 +1803,46 @@ public static String randomDoc() {
"}";
}

public static String randomNetworkDoc() {
return "{\n" +
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
"\"Keywords\":\"9223372036854775808\",\n" +
"\"SeverityValue\":2,\n" +
"\"Severity\":\"INFO\",\n" +
"\"EventID\":22,\n" +
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
"\"SourceIp\":\"1.2.3.4\",\n" +
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
"\"Version\":5,\n" +
"\"TaskValue\":22,\n" +
"\"OpcodeValue\":0,\n" +
"\"RecordNumber\":9532,\n" +
"\"ExecutionProcessID\":1996,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
"\"Domain\":\"NTAUTHORITY\",\n" +
"\"AccountName\":\"SYSTEM\",\n" +
"\"UserID\":\"S-1-5-18\",\n" +
"\"AccountType\":\"User\",\n" +
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
"\"Opcode\":\"Info\",\n" +
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
"\"QueryResults\":\"172.31.46.38;\",\n" +
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
"\"SourceModuleName\":\"in\",\n" +
"\"SourceModuleType\":\"im_msvistalog\",\n" +
"\"CommandLine\": \"eachtest\",\n" +
"\"id.orig_h\": \"123.12.123.12\",\n" +
"\"Initiated\": \"true\"\n" +
"}";
}

public static String randomCloudtrailAggrDoc(String eventType, String accountId) {
return "{\n" +
" \"AccountName\": \"" + accountId + "\",\n" +
Expand All @@ -1791,6 +1860,7 @@ public static String randomVpcFlowDoc() {
" \"srcport\": 9000,\n" +
" \"dstport\": 8000,\n" +
" \"severity_id\": \"-1\",\n" +
" \"id.orig_h\": \"1.2.3.4\",\n" +
" \"class_name\": \"Network Activity\"\n" +
"}";
}
Expand Down Expand Up @@ -2366,7 +2436,7 @@ public static List<String> randomLowerCaseStringList() {
stringList.add(randomLowerCaseString());
return stringList;
}

public static XContentParser parser(String xc) throws IOException {
XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc);
parser.nextToken();
Expand Down
46 changes: 46 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
assertEquals(24, unmappedFieldAliases.size());
}

@SuppressWarnings("unchecked")
public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException {
String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings());

Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
// both req params and req body are supported
request.addParameter("index_name", index);
request.addParameter("rule_topic", "cloudtrail");
Response response = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
Map<String, Object> respMap = responseAsMap(response);
// Verify alias mappings
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
Assert.assertEquals(18, props.size());
// Verify unmapped index fields
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases.size());

// create a cloudtrail rule with a raw field
String rule = randomRuleWithRawField();
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));

// check the mapping view API again to ensure it's the same after rule is created
Response response2 = client().performRequest(request);
assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode());
Map<String, Object> respMap2 = responseAsMap(response2);
// Verify alias mappings
Map<String, Object> props2 = (Map<String, Object>) respMap2.get("properties");
Assert.assertEquals(18, props2.size());
// Verify unmapped index fields
List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
assertEquals(20, unmappedIndexFields2.size());
// Verify unmapped field aliases
List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
assertEquals(25, unmappedFieldAliases2.size());
// Verify that first response and second response are the same after rule was indexed
assertEquals(props, props2);
assertEquals(unmappedIndexFields, unmappedIndexFields2);
assertEquals(unmappedFieldAliases, unmappedFieldAliases2);
}

@SuppressWarnings("unchecked")
public void testOCSFVpcflowGetMappingsViewApi() throws IOException {
String index = createTestIndex("vpcflow", ocsfVpcflowMappings());
Expand Down

0 comments on commit 865ad34

Please sign in to comment.