Skip to content

Commit

Permalink
Merge branch 'feature/threat_intel' into 3.0-threat-intel-dev
Browse files Browse the repository at this point in the history
Signed-off-by: AWSHurneyt <[email protected]>
  • Loading branch information
AWSHurneyt authored May 31, 2024
2 parents 5429ac9 + 243a0ca commit c9f0bec
Show file tree
Hide file tree
Showing 34 changed files with 1,944 additions and 141 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
import org.opensearch.commons.alerting.action.AlertingActions;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.env.Environment;
import org.opensearch.env.NodeEnvironment;
import org.opensearch.index.IndexSettings;
Expand Down Expand Up @@ -63,13 +65,21 @@
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.resthandler.*;
import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader;
import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner;
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService;
Expand All @@ -87,6 +97,7 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD;
import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension {
Expand All @@ -103,11 +114,15 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate";
public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations";
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules";
public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel";
public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source";
public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc";
public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch";

public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype";
public static final String JOB_INDEX_NAME = ".opensearch-sap--job";
public static final String JOB_TYPE = "opensearch_sap_job";

public static final Map<String, Object> TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true);

private CorrelationRuleIndices correlationRuleIndices;
Expand All @@ -131,6 +146,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
private BuiltinLogTypeLoader builtinLogTypeLoader;

private LogTypeService logTypeService;

private SATIFSourceConfigDao SaTifSourceConfigDao;

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings){
return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
Expand Down Expand Up @@ -167,13 +185,16 @@ public Collection<Object> createComponents(Client client,
TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService);
TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader);
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);
SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool, xContentRegistry, threatIntelLockService);
SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService);


TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);

return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
tifJobUpdateService, tifJobParameterService, threatIntelLockService);
tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigDao, SaTifSourceConfigService);
}

@Override
Expand Down Expand Up @@ -213,13 +234,15 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchCorrelationRuleAction(),
new RestIndexCustomLogTypeAction(),
new RestSearchCustomLogTypeAction(),
new RestDeleteCustomLogTypeAction()
new RestDeleteCustomLogTypeAction(),
new RestIndexTIFSourceConfigAction(),
new RestGetTIFSourceConfigAction()
);
}

@Override
public String getJobType() {
return "opensearch_sap_job";
return JOB_TYPE;
}

@Override
Expand All @@ -234,7 +257,21 @@ public ScheduledJobRunner getJobRunner() {

@Override
public ScheduledJobParser getJobParser() {
return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null);
return (xcp, id, jobDocVersion) -> {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();
switch (fieldName) {
case FEED_SOURCE_CONFIG_FIELD:
return SATIFSourceConfig.parse(xcp, id, null);
default:
log.error("Job parser failed for [{}] in security analytics job registration", fieldName);
xcp.skipChildren();
}
}
return null;
};
}

@Override
Expand Down Expand Up @@ -334,7 +371,9 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class),
new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class),
new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class),
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class)
new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class),
new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class),
new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class)
);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionType;

import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.GET_TIF_SOURCE_CONFIG_ACTION_NAME;

/**
* Get TIF Source Config Action
*/
public class SAGetTIFSourceConfigAction extends ActionType<SAGetTIFSourceConfigResponse> {

public static final SAGetTIFSourceConfigAction INSTANCE = new SAGetTIFSourceConfigAction();
public static final String NAME = GET_TIF_SOURCE_CONFIG_ACTION_NAME;
private SAGetTIFSourceConfigAction() {
super(NAME, SAGetTIFSourceConfigResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;

import java.io.IOException;
import java.util.Locale;

import static org.opensearch.action.ValidateActions.addValidationError;

/**
* Get threat intel feed source config request
*/
public class SAGetTIFSourceConfigRequest extends ActionRequest {
private final String id;
private final Long version;
public static final String TIF_SOURCE_CONFIG_ID = "tif_source_config_id";

public SAGetTIFSourceConfigRequest(String id, Long version) {
super();
this.id = id;
this.version = version;
}

public SAGetTIFSourceConfigRequest(StreamInput sin) throws IOException {
this(sin.readString(), // id
sin.readLong()); // version
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeLong(version);
}

public String getId() {
return id;
}

public Long getVersion() {
return version;
}


@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (id == null || id.isEmpty()) {
validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", TIF_SOURCE_CONFIG_ID), validationException);
}
return validationException;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;

import java.io.IOException;

import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID;
import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION;

public class SAGetTIFSourceConfigResponse extends ActionResponse implements ToXContentObject {
private final String id;

private final Long version;

private final RestStatus status;

private final SATIFSourceConfigDto SaTifSourceConfigDto;


public SAGetTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) {
super();
this.id = id;
this.version = version;
this.status = status;
this.SaTifSourceConfigDto = SaTifSourceConfigDto;
}

public SAGetTIFSourceConfigResponse(StreamInput sin) throws IOException {
this(
sin.readString(), // id
sin.readLong(), // version
sin.readEnum(RestStatus.class), // status
sin.readBoolean()? SATIFSourceConfigDto.readFrom(sin) : null // SA tif config dto
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeLong(version);
out.writeEnum(status);
if (SaTifSourceConfigDto != null) {
out.writeBoolean((true));
SaTifSourceConfigDto.writeTo(out);
} else {
out.writeBoolean(false);
}
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field(_ID, id)
.field(_VERSION, version);
builder.startObject("tif_config")
.field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName())
.field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat())
.field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType())
.field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState())
.field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime())
.field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled())
.field(SATIFSourceConfigDto.CREATED_AT_FIELD, SaTifSourceConfigDto.getCreatedAt())
.field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, SaTifSourceConfigDto.getLastUpdateTime())
.field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime())
.field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, SaTifSourceConfigDto.getRefreshType())
.field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, SaTifSourceConfigDto.getLastRefreshedUser())
.field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule())
// source
.field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser())
.field(SATIFSourceConfigDto.IOC_MAP_STORE_FIELD, SaTifSourceConfigDto.getIocMapStore())
.field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes())
.endObject();
return builder.endObject();
}

public String getId() {
return id;
}

public Long getVersion() {
return version;
}

public RestStatus getStatus() {
return status;
}

public SATIFSourceConfigDto getSaTifSourceConfigDto() {
return SaTifSourceConfigDto;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionType;

import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME;

/**
* Threat intel tif job creation action
*/
public class SAIndexTIFSourceConfigAction extends ActionType<SAIndexTIFSourceConfigResponse> {

public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction();
public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME;
private SAIndexTIFSourceConfigAction() {
super(NAME, SAIndexTIFSourceConfigResponse::new);
}
}
Loading

0 comments on commit c9f0bec

Please sign in to comment.