diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index cf1ce83a1..9c040d839 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -30,6 +30,8 @@ import org.opensearch.commons.alerting.action.AlertingActions; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; @@ -63,13 +65,21 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; @@ -87,6 +97,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.FEED_SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { @@ -103,11 +114,15 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; + public static final String THREAT_INTEL_BASE_URI = PLUGINS_BASE_URI + "/threat_intel"; + public static final String THREAT_INTEL_SOURCE_URI = PLUGINS_BASE_URI + "/threat_intel/source"; public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc"; public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; + public static final String JOB_TYPE = "opensearch_sap_job"; + public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); private CorrelationRuleIndices correlationRuleIndices; @@ -131,6 +146,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private BuiltinLogTypeLoader builtinLogTypeLoader; private LogTypeService logTypeService; + + private SATIFSourceConfigDao SaTifSourceConfigDao; + @Override public Collection getSystemIndexDescriptors(Settings settings){ return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); @@ -167,13 +185,16 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); + SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); + SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService); + TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigDao, SaTifSourceConfigService); } @Override @@ -213,13 +234,15 @@ public List getRestHandlers(Settings settings, new RestSearchCorrelationRuleAction(), new RestIndexCustomLogTypeAction(), new RestSearchCustomLogTypeAction(), - new RestDeleteCustomLogTypeAction() + new RestDeleteCustomLogTypeAction(), + new RestIndexTIFSourceConfigAction(), + new RestGetTIFSourceConfigAction() ); } @Override public String getJobType() { - return "opensearch_sap_job"; + return JOB_TYPE; } @Override @@ -234,7 +257,21 @@ public ScheduledJobRunner getJobRunner() { @Override public ScheduledJobParser getJobParser() { - return (parser, id, jobDocVersion) -> TIFJobParameter.PARSER.parse(parser, null); + return (xcp, id, jobDocVersion) -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + return SATIFSourceConfig.parse(xcp, id, null); + default: + log.error("Job parser failed for [{}] in security analytics job registration", fieldName); + xcp.skipChildren(); + } + } + return null; + }; } @Override @@ -334,7 +371,9 @@ public List> getSettings() { new ActionHandler<>(IndexCustomLogTypeAction.INSTANCE, TransportIndexCustomLogTypeAction.class), new ActionHandler<>(SearchCustomLogTypeAction.INSTANCE, TransportSearchCustomLogTypeAction.class), new ActionHandler<>(DeleteCustomLogTypeAction.INSTANCE, TransportDeleteCustomLogTypeAction.class), - new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class) + new ActionHandler<>(PutTIFJobAction.INSTANCE, TransportPutTIFJobAction.class), + new ActionHandler<>(SAIndexTIFSourceConfigAction.INSTANCE, TransportIndexTIFSourceConfigAction.class), + new ActionHandler<>(SAGetTIFSourceConfigAction.INSTANCE, TransportGetTIFSourceConfigAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java new file mode 100644 index 000000000..f2a0099e7 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.GET_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Get TIF Source Config Action + */ +public class SAGetTIFSourceConfigAction extends ActionType { + + public static final SAGetTIFSourceConfigAction INSTANCE = new SAGetTIFSourceConfigAction(); + public static final String NAME = GET_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAGetTIFSourceConfigAction() { + super(NAME, SAGetTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java new file mode 100644 index 000000000..6f64809bd --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.action.ValidateActions.addValidationError; + +/** + * Get threat intel feed source config request + */ +public class SAGetTIFSourceConfigRequest extends ActionRequest { + private final String id; + private final Long version; + public static final String TIF_SOURCE_CONFIG_ID = "tif_source_config_id"; + + public SAGetTIFSourceConfigRequest(String id, Long version) { + super(); + this.id = id; + this.version = version; + } + + public SAGetTIFSourceConfigRequest(StreamInput sin) throws IOException { + this(sin.readString(), // id + sin.readLong()); // version + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (id == null || id.isEmpty()) { + validationException = addValidationError(String.format(Locale.getDefault(), "%s is missing", TIF_SOURCE_CONFIG_ID), validationException); + } + return validationException; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java new file mode 100644 index 000000000..e239b87af --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java @@ -0,0 +1,101 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAGetTIFSourceConfigResponse extends ActionResponse implements ToXContentObject { + private final String id; + + private final Long version; + + private final RestStatus status; + + private final SATIFSourceConfigDto SaTifSourceConfigDto; + + + public SAGetTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAGetTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + sin.readBoolean()? SATIFSourceConfigDto.readFrom(sin) : null // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + if (SaTifSourceConfigDto != null) { + out.writeBoolean((true)); + SaTifSourceConfigDto.writeTo(out); + } else { + out.writeBoolean(false); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + builder.startObject("tif_config") + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.CREATED_AT_FIELD, SaTifSourceConfigDto.getCreatedAt()) + .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, SaTifSourceConfigDto.getLastUpdateTime()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, SaTifSourceConfigDto.getRefreshType()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, SaTifSourceConfigDto.getLastRefreshedUser()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule()) + // source + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_MAP_STORE_FIELD, SaTifSourceConfigDto.getIocMapStore()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes()) + .endObject(); + return builder.endObject(); + } + + public String getId() { + return id; + } + + public Long getVersion() { + return version; + } + + public RestStatus getStatus() { + return status; + } + + public SATIFSourceConfigDto getSaTifSourceConfigDto() { + return SaTifSourceConfigDto; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..1b4acd80e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigAction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Threat intel tif job creation action + */ +public class SAIndexTIFSourceConfigAction extends ActionType { + + public static final SAIndexTIFSourceConfigAction INSTANCE = new SAIndexTIFSourceConfigAction(); + public static final String NAME = INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + private SAIndexTIFSourceConfigAction() { + super(NAME, SAIndexTIFSourceConfigResponse::new); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..a44a412ae --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigRequest.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.common.ParameterValidator; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigRequest; + +import java.io.IOException; +import java.util.List; + +/** + * Threat intel feed config creation request + */ +public class SAIndexTIFSourceConfigRequest extends ActionRequest implements IndexTIFSourceConfigRequest { + private static final ParameterValidator VALIDATOR = new ParameterValidator(); + private String tifSourceConfigId; + private final WriteRequest.RefreshPolicy refreshPolicy; + private final RestRequest.Method method; + private SATIFSourceConfigDto SaTifSourceConfigDto; + + public SAIndexTIFSourceConfigRequest(String tifSourceConfigId, + WriteRequest.RefreshPolicy refreshPolicy, + RestRequest.Method method, + SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.tifSourceConfigId = tifSourceConfigId; + this.refreshPolicy = refreshPolicy; + this.method = method; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigRequest(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + WriteRequest.RefreshPolicy.readFrom(sin), // refresh policy + sin.readEnum(RestRequest.Method.class), // method + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(tifSourceConfigId); + refreshPolicy.writeTo(out); + out.writeEnum(method); + SaTifSourceConfigDto.writeTo(out); + } + + @Override + public String getTIFConfigId() { + return tifSourceConfigId; + } + + public void setTIFConfigId(String tifConfigId) { + this.tifSourceConfigId = tifConfigId; + } + + @Override + public SATIFSourceConfigDto getTIFConfigDto() { + return SaTifSourceConfigDto; + } + + public void setTIFConfigDto(SATIFSourceConfigDto SaTifSourceConfigDto) { + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + public RestRequest.Method getMethod() { + return method; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException errors = new ActionRequestValidationException(); + List errorMsgs = VALIDATOR.validateTIFJobName(SaTifSourceConfigDto.getName()); + if (errorMsgs.isEmpty() == false) { + errorMsgs.forEach(errors::addValidationError); + } + return errors.validationErrors().isEmpty() ? null : errors; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..b4ea1b9c0 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; +import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; + +public class SAIndexTIFSourceConfigResponse extends ActionResponse implements ToXContentObject, IndexTIFSourceConfigResponse { + private final String id; + private final Long version; + private final RestStatus status; + private final SATIFSourceConfigDto SaTifSourceConfigDto; + + public SAIndexTIFSourceConfigResponse(String id, Long version, RestStatus status, SATIFSourceConfigDto SaTifSourceConfigDto) { + super(); + this.id = id; + this.version = version; + this.status = status; + this.SaTifSourceConfigDto = SaTifSourceConfigDto; + } + + public SAIndexTIFSourceConfigResponse(StreamInput sin) throws IOException { + this( + sin.readString(), // tif config id + sin.readLong(), // version + sin.readEnum(RestStatus.class), // status + SATIFSourceConfigDto.readFrom(sin) // SA tif config dto + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeEnum(status); + SaTifSourceConfigDto.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(_ID, id) + .field(_VERSION, version); + + builder.startObject("tif_config") + .field(SATIFSourceConfigDto.FEED_FORMAT_FIELD, SaTifSourceConfigDto.getFeedFormat()) + .field(SATIFSourceConfigDto.FEED_NAME_FIELD, SaTifSourceConfigDto.getName()) + .field(SATIFSourceConfigDto.FEED_TYPE_FIELD, SaTifSourceConfigDto.getFeedType()) + .field(SATIFSourceConfigDto.STATE_FIELD, SaTifSourceConfigDto.getState()) + .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, SaTifSourceConfigDto.getEnabledTime()) + .field(SATIFSourceConfigDto.ENABLED_FIELD, SaTifSourceConfigDto.isEnabled()) + .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, SaTifSourceConfigDto.getLastRefreshedTime()) + .field(SATIFSourceConfigDto.SCHEDULE_FIELD, SaTifSourceConfigDto.getSchedule()) + // source + .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, SaTifSourceConfigDto.getCreatedByUser()) + .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, SaTifSourceConfigDto.getIocTypes()) + .endObject(); + + return builder.endObject(); + } + @Override + public String getTIFConfigId() { + return id; + } + @Override + public Long getVersion() { + return version; + } + @Override + public TIFSourceConfigDto getTIFConfigDto() { + return SaTifSourceConfigDto; + } + public RestStatus getStatus() { + return status; + } + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java new file mode 100644 index 000000000..0ac915781 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/RefreshType.java @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Refresh Types: Full + * TODO: Add other refresh types such as the delta + */ +public enum RefreshType { + + FULL +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java new file mode 100644 index 000000000..3e3cfa311 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java @@ -0,0 +1,216 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * CRUD for threat intel feeds source config object + */ +public class SATIFSourceConfigDao { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDao.class); + private final Client client; + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final ThreadPool threadPool; + private final NamedXContentRegistry xContentRegistry; + private final TIFLockService lockService; + + + public SATIFSourceConfigDao(final Client client, + final ClusterService clusterService, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + final TIFLockService lockService + ) { + this.client = client; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.threadPool = threadPool; + this.xContentRegistry = xContentRegistry; + this.lockService = lockService; + } + + public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, + TimeValue indexTimeout, + final LockModel lock, + final ActionListener actionListener + ) { + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(SaTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(SaTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, actionListener::onFailure)); + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + }, exception -> { + lockService.releaseLock(lock); + log.error("Failed to release lock", exception); + actionListener.onFailure(exception); + }); + createJobIndexIfNotExists(createIndexStepListener); + } + + private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, IndexResponse response) { + return new SATIFSourceConfig( + response.getId(), + SaTifSourceConfig.getVersion(), + SaTifSourceConfig.getName(), + SaTifSourceConfig.getFeedFormat(), + SaTifSourceConfig.getFeedType(), + SaTifSourceConfig.getCreatedByUser(), + SaTifSourceConfig.getCreatedAt(), + SaTifSourceConfig.getEnabledTime(), + SaTifSourceConfig.getLastUpdateTime(), + SaTifSourceConfig.getSchedule(), + SaTifSourceConfig.getState(), + SaTifSourceConfig.getRefreshType(), + SaTifSourceConfig.getLastRefreshedTime(), + SaTifSourceConfig.getLastRefreshedUser(), + SaTifSourceConfig.isEnabled(), + SaTifSourceConfig.getIocMapStore(), + SaTifSourceConfig.getIocTypes() + ); + } + + // Get the job config index mapping + private String getIndexMapping() { + try { + try (InputStream is = SATIFSourceConfigDao.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel index mapping", e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + // Create TIF source config index + /** + * Index name: .opensearch-sap--job + * Mapping: /mappings/threat_intel_job_mapping.json + * + * @param stepListener setup listener + */ + public void createJobIndexIfNotExists(final StepListener stepListener) { + // check if job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { + stepListener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(final CreateIndexResponse createIndexResponse) { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); + } + })); + } + + + // Get TIF source config + public void getTIFSourceConfig( + String tifSourceConfigId, + Long version, + ActionListener actionListener + ) { + GetRequest getRequest = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, tifSourceConfigId).version(version); + client.get(getRequest, new ActionListener<>() { + @Override + public void onResponse(GetResponse response) { + try { + if (!response.isExists()) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND))); + return; + } + SATIFSourceConfig SaTifSourceConfig = null; + if (!response.isSourceEmpty()) { + XContentParser xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef(), XContentType.JSON + ); + SaTifSourceConfig = SATIFSourceConfig.docParse(xcp, response.getId(), response.getVersion()); + assert SaTifSourceConfig != null; + } + log.debug("Threat intel source config with id [{}] fetched.", response.getId()); + actionListener.onResponse(SaTifSourceConfig); + } catch (IOException ex) { + log.error("Failed to fetch threat intel source config document", ex); + actionListener.onFailure(ex); + } + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch threat intel source config document " + tifSourceConfigId, e); + actionListener.onFailure(e); + } + }); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 46f576b4e..dc9420381 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -19,14 +18,17 @@ import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; -import org.opensearch.jobscheduler.spi.schedule.Schedule; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfig; import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -37,14 +39,13 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ private static final Logger log = LogManager.getLogger(SATIFSourceConfig.class); - /** * Prefix of indices having threatIntel data */ public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threat-intel"; + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; public static final String NO_ID = ""; - public static final String ID_FIELD = "id"; public static final Long NO_VERSION = 1L; public static final String VERSION_FIELD = "version"; @@ -63,6 +64,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; public static final String ENABLED_FIELD = "enabled"; public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; private String id; private Long version; @@ -75,17 +77,18 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ // private Source source; TODO: create Source Object private Instant enabledTime; private Instant lastUpdateTime; - private Schedule schedule; + private IntervalSchedule schedule; private TIFJobState state; - public String refreshType; + public RefreshType refreshType; public Instant lastRefreshedTime; public String lastRefreshedUser; private Boolean isEnabled; private Map iocMapStore; + private List iocTypes; public SATIFSourceConfig(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, - Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, - Boolean isEnabled, Map iocMapStore) { + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.feedName = feedName; @@ -94,9 +97,9 @@ public SATIFSourceConfig(String id, Long version, String feedName, String feedFo this.createdByUser = createdByUser; this.createdAt = createdAt != null ? createdAt : Instant.now(); - if (this.isEnabled == null && this.enabledTime == null) { + if (isEnabled == null && enabledTime == null) { this.enabledTime = Instant.now(); - } else if (this.isEnabled != null && !this.isEnabled) { + } else if (isEnabled != null && !isEnabled) { this.enabledTime = null; } else { this.enabledTime = enabledTime; @@ -104,14 +107,13 @@ public SATIFSourceConfig(String id, Long version, String feedName, String feedFo this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); this.schedule = schedule; - - this.state = (this.state == null) ? TIFJobState.CREATING : state; - - this.refreshType = refreshType; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; this.lastRefreshedTime = lastRefreshedTime; this.lastRefreshedUser = lastRefreshedUser; this.isEnabled = isEnabled; - this.iocMapStore = iocMapStore; + this.iocMapStore = iocMapStore != null ? iocMapStore : new HashMap<>(); + this.iocTypes = iocTypes; } public SATIFSourceConfig(StreamInput sin) throws IOException { @@ -121,17 +123,18 @@ public SATIFSourceConfig(StreamInput sin) throws IOException { sin.readString(), // feed name sin.readString(), // feed format FeedType.valueOf(sin.readString()), // feed type - sin.readString(), // created by user + sin.readOptionalString(), // created by user sin.readInstant(), // created at - sin.readInstant(), // enabled time + sin.readOptionalInstant(), // enabled time sin.readInstant(), // last update time new IntervalSchedule(sin), // schedule TIFJobState.valueOf(sin.readString()), // state - sin.readString(), // refresh type + RefreshType.valueOf(sin.readString()), // state sin.readOptionalInstant(), // last refreshed time sin.readOptionalString(), // last refreshed user sin.readBoolean(), // is enabled - sin.readMap() // ioc map store + sin.readMap(), // ioc map store + sin.readStringList() ); } @@ -141,28 +144,29 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(feedName); out.writeString(feedFormat); out.writeString(feedType.name()); - out.writeString(createdByUser); + out.writeOptionalString(createdByUser); out.writeInstant(createdAt); - out.writeInstant(enabledTime); + out.writeOptionalInstant(enabledTime); out.writeInstant(lastUpdateTime); schedule.writeTo(out); out.writeString(state.name()); - out.writeString(refreshType); - out.writeOptionalInstant(lastRefreshedTime == null ? null : lastRefreshedTime); - out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); + out.writeString(refreshType.name()); + out.writeOptionalInstant(lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser); out.writeBoolean(isEnabled); out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.field(ID_FIELD, id); - builder.field(VERSION_FIELD, version); - builder.field(FEED_NAME_FIELD, feedName); - builder.field(FEED_FORMAT_FIELD, feedFormat); - builder.field(FEED_TYPE_FIELD, feedType.name()); - builder.field(CREATED_BY_USER_FIELD, createdByUser); + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); if (createdAt == null) { builder.nullField(CREATED_AT_FIELD); @@ -184,7 +188,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SCHEDULE_FIELD, schedule); builder.field(STATE_FIELD, state.name()); - builder.field(REFRESH_TYPE_FIELD, refreshType); + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); if (lastRefreshedTime == null) { builder.nullField(LAST_REFRESHED_TIME_FIELD); } else { @@ -194,10 +198,24 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); builder.field(ENABLED_FIELD, isEnabled); builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); builder.endObject(); return builder; } + public static SATIFSourceConfig docParse(XContentParser xcp, String id, Long version) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + SATIFSourceConfig SaTifSourceConfig = parse(xcp, id, version); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp); + + SaTifSourceConfig.setId(id); + SaTifSourceConfig.setVersion(version); + return SaTifSourceConfig; + } + public static SATIFSourceConfig parse(XContentParser xcp, String id, Long version) throws IOException { if (id == null) { id = NO_ID; @@ -213,21 +231,23 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio Instant createdAt = null; Instant enabledTime = null; Instant lastUpdateTime = null; - Schedule schedule = null; + IntervalSchedule schedule = null; TIFJobState state = null; - String refreshType = null; + RefreshType refreshType = null; Instant lastRefreshedTime = null; String lastRefreshedUser = null; Boolean isEnabled = null; Map iocMapStore = null; + List iocTypes = new ArrayList<>(); - xcp.nextToken(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; case FEED_NAME_FIELD: feedName = xcp.text(); break; @@ -275,7 +295,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio } break; case SCHEDULE_FIELD: - schedule = ScheduleParser.parse(xcp); + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); break; case STATE_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -288,7 +308,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { refreshType = null; } else { - refreshType = xcp.text(); + refreshType = toRefreshType(xcp.text()); } break; case LAST_REFRESHED_TIME_FIELD: @@ -312,9 +332,18 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio isEnabled = xcp.booleanValue(); break; case IOC_MAP_STORE_FIELD: - iocMapStore = xcp.map(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } break; - default: xcp.skipChildren(); } @@ -342,7 +371,8 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio lastRefreshedTime, lastRefreshedUser, isEnabled, - iocMapStore + iocMapStore, + iocTypes ); } @@ -365,6 +395,15 @@ public static FeedType toFeedType(String feedType) { } } + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + return null; + } + } + public static SATIFSourceConfig readFrom(StreamInput sin) throws IOException { return new SATIFSourceConfig(sin); } @@ -424,10 +463,10 @@ public Instant getLastUpdateTime() { public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public Schedule getSchedule() { + public IntervalSchedule getSchedule() { return this.schedule; } - public void setSchedule(Schedule schedule) { + public void setSchedule(IntervalSchedule schedule) { this.schedule = schedule; } public TIFJobState getState() { @@ -448,10 +487,10 @@ public Instant getLastRefreshedTime() { public void setLastRefreshedTime(Instant lastRefreshedTime) { this.lastRefreshedTime = lastRefreshedTime; } - public String getRefreshType() { + public RefreshType getRefreshType() { return refreshType; } - public void setRefreshType(String refreshType) { + public void setRefreshType(RefreshType refreshType) { this.refreshType = refreshType; } public boolean isEnabled() { @@ -474,4 +513,12 @@ public Map getIocMapStore() { public void setIocMapStore(Map iocMapStore) { this.iocMapStore = iocMapStore; } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index c8344e5e1..89dc80d17 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -18,15 +17,18 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -37,9 +39,9 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou private static final Logger log = LogManager.getLogger(SATIFSourceConfigDto.class); + public static final String FEED_SOURCE_CONFIG_FIELD = "feed_source_config"; public static final String NO_ID = ""; - public static final String ID_FIELD = "id"; public static final Long NO_VERSION = 1L; public static final String VERSION_FIELD = "version"; @@ -58,6 +60,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; public static final String ENABLED_FIELD = "enabled"; public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + public static final String IOC_TYPES_FIELD = "ioc_types"; private String id; private Long version; @@ -70,36 +73,38 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou // private Source source; TODO: create Source Object private Instant enabledTime; private Instant lastUpdateTime; - private Schedule schedule; + private IntervalSchedule schedule; private TIFJobState state; - public String refreshType; + public RefreshType refreshType; public Instant lastRefreshedTime; public String lastRefreshedUser; private Boolean isEnabled; private Map iocMapStore; - - public SATIFSourceConfigDto(SATIFSourceConfig saTIFSourceConfig) { - this.id = saTIFSourceConfig.getId(); - this.version = saTIFSourceConfig.getVersion(); - this.feedName = saTIFSourceConfig.getName(); - this.feedFormat = saTIFSourceConfig.getFeedFormat(); - this.feedType = saTIFSourceConfig.getFeedType(); - this.createdByUser = saTIFSourceConfig.getCreatedByUser(); - this.createdAt = saTIFSourceConfig.getCreatedAt(); - this.enabledTime = saTIFSourceConfig.getEnabledTime(); - this.lastUpdateTime = saTIFSourceConfig.getLastUpdateTime(); - this.schedule = saTIFSourceConfig.getSchedule(); - this.state = saTIFSourceConfig.getState();; - this.refreshType = saTIFSourceConfig.getRefreshType(); - this.lastRefreshedTime = saTIFSourceConfig.getLastRefreshedTime(); - this.lastRefreshedUser = saTIFSourceConfig.getLastRefreshedUser(); - this.isEnabled = saTIFSourceConfig.isEnabled();; - this.iocMapStore = saTIFSourceConfig.getIocMapStore(); + private List iocTypes; + + public SATIFSourceConfigDto(SATIFSourceConfig SaTifSourceConfig) { + this.id = SaTifSourceConfig.getId(); + this.version = SaTifSourceConfig.getVersion(); + this.feedName = SaTifSourceConfig.getName(); + this.feedFormat = SaTifSourceConfig.getFeedFormat(); + this.feedType = SaTifSourceConfig.getFeedType(); + this.createdByUser = SaTifSourceConfig.getCreatedByUser(); + this.createdAt = SaTifSourceConfig.getCreatedAt(); + this.enabledTime = SaTifSourceConfig.getEnabledTime(); + this.lastUpdateTime = SaTifSourceConfig.getLastUpdateTime(); + this.schedule = SaTifSourceConfig.getSchedule(); + this.state = SaTifSourceConfig.getState();; + this.refreshType = SaTifSourceConfig.getRefreshType(); + this.lastRefreshedTime = SaTifSourceConfig.getLastRefreshedTime(); + this.lastRefreshedUser = SaTifSourceConfig.getLastRefreshedUser(); + this.isEnabled = SaTifSourceConfig.isEnabled();; + this.iocMapStore = SaTifSourceConfig.getIocMapStore(); + this.iocTypes = SaTifSourceConfig.getIocTypes(); } public SATIFSourceConfigDto(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, - Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, - Boolean isEnabled, Map iocMapStore) { + Instant enabledTime, Instant lastUpdateTime, IntervalSchedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore, List iocTypes) { this.id = id != null ? id : NO_ID; this.version = version != null ? version : NO_VERSION; this.feedName = feedName; @@ -108,9 +113,9 @@ public SATIFSourceConfigDto(String id, Long version, String feedName, String fee this.createdByUser = createdByUser; this.createdAt = createdAt != null ? createdAt : Instant.now(); - if (this.isEnabled == null && this.enabledTime == null) { + if (isEnabled == null && enabledTime == null) { this.enabledTime = Instant.now(); - } else if (this.isEnabled != null && !this.isEnabled) { + } else if (isEnabled != null && !isEnabled) { this.enabledTime = null; } else { this.enabledTime = enabledTime; @@ -118,14 +123,13 @@ public SATIFSourceConfigDto(String id, Long version, String feedName, String fee this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); this.schedule = schedule; - - this.state = (this.state == null) ? TIFJobState.CREATING : state; - - this.refreshType = refreshType; + this.state = state != null ? state : TIFJobState.CREATING; + this.refreshType = refreshType != null ? refreshType : RefreshType.FULL; this.lastRefreshedTime = lastRefreshedTime; this.lastRefreshedUser = lastRefreshedUser; this.isEnabled = isEnabled; - this.iocMapStore = (this.iocMapStore == null) ? new HashMap<>() : iocMapStore; + this.iocMapStore = iocMapStore != null ? iocMapStore : new HashMap<>(); + this.iocTypes = iocTypes; } public SATIFSourceConfigDto(StreamInput sin) throws IOException { @@ -138,28 +142,29 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(feedName); out.writeString(feedFormat); out.writeString(feedType.name()); - out.writeString(createdByUser); + out.writeOptionalString(createdByUser); out.writeInstant(createdAt); - out.writeInstant(enabledTime); + out.writeOptionalInstant(enabledTime); out.writeInstant(lastUpdateTime); schedule.writeTo(out); out.writeString(state.name()); - out.writeOptionalString(refreshType == null? null: refreshType); - out.writeOptionalInstant(lastRefreshedTime == null ? null : lastRefreshedTime); - out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); + out.writeString(refreshType.name()); + out.writeOptionalInstant(lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser); out.writeBoolean(isEnabled); out.writeMap(iocMapStore); + out.writeStringCollection(iocTypes); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.field(ID_FIELD, id); - builder.field(VERSION_FIELD, version); - builder.field(FEED_NAME_FIELD, feedName); - builder.field(FEED_FORMAT_FIELD, feedFormat); - builder.field(FEED_TYPE_FIELD, feedType); - builder.field(CREATED_BY_USER_FIELD, createdByUser); + builder.startObject() + .startObject(FEED_SOURCE_CONFIG_FIELD) + .field(VERSION_FIELD, version) + .field(FEED_NAME_FIELD, feedName) + .field(FEED_FORMAT_FIELD, feedFormat) + .field(FEED_TYPE_FIELD, feedType.name()) + .field(CREATED_BY_USER_FIELD, createdByUser); if (createdAt == null) { builder.nullField(CREATED_AT_FIELD); @@ -181,30 +186,19 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(SCHEDULE_FIELD, schedule); builder.field(STATE_FIELD, state.name()); - - if (refreshType == null) { - builder.nullField(REFRESH_TYPE_FIELD); - } else { - builder.field(REFRESH_TYPE_FIELD, refreshType); - } - + builder.field(REFRESH_TYPE_FIELD, refreshType.name()); if (lastRefreshedTime == null) { builder.nullField(LAST_REFRESHED_TIME_FIELD); } else { builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); } - - if (lastRefreshedUser == null) { - builder.nullField(LAST_REFRESHED_USER_FIELD); - } else { - builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); - } builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); builder.field(ENABLED_FIELD, isEnabled); builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.field(IOC_TYPES_FIELD, iocTypes); + builder.endObject(); builder.endObject(); - return builder; } @@ -223,21 +217,22 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver Instant createdAt = null; Instant enabledTime = null; Instant lastUpdateTime = null; - Schedule schedule = null; + IntervalSchedule schedule = null; TIFJobState state = null; - String refreshType = null; + RefreshType refreshType = null; Instant lastRefreshedTime = null; String lastRefreshedUser = null; Boolean isEnabled = null; - Map iocMapStore = new HashMap<>(); + Map iocMapStore = null; + List iocTypes = null; - xcp.nextToken(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); xcp.nextToken(); - switch (fieldName) { + case FEED_SOURCE_CONFIG_FIELD: + break; case FEED_NAME_FIELD: feedName = xcp.text(); break; @@ -248,7 +243,11 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver feedType = toFeedType(xcp.text()); break; case CREATED_BY_USER_FIELD: - createdByUser = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = xcp.text(); + } break; case CREATED_AT_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -281,7 +280,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case SCHEDULE_FIELD: - schedule = ScheduleParser.parse(xcp); + schedule = (IntervalSchedule) ScheduleParser.parse(xcp); break; case STATE_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -291,7 +290,11 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case REFRESH_TYPE_FIELD: - refreshType = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = toRefreshType(xcp.text()); + } break; case LAST_REFRESHED_TIME_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -304,15 +307,29 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver } break; case LAST_REFRESHED_USER_FIELD: - lastRefreshedUser = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = xcp.text(); + } break; case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; case IOC_MAP_STORE_FIELD: - iocMapStore = xcp.map(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + iocMapStore = null; + } else { + iocMapStore = xcp.map(); + } + break; + case IOC_TYPES_FIELD: + iocTypes = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + iocTypes.add(xcp.text()); + } break; - default: xcp.skipChildren(); } @@ -340,7 +357,8 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver lastRefreshedTime, lastRefreshedUser, isEnabled, - iocMapStore + iocMapStore, + iocTypes ); } @@ -363,6 +381,15 @@ public static FeedType toFeedType(String feedType) { } } + public static RefreshType toRefreshType(String stateName) { + try { + return RefreshType.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid refresh type, cannot be parsed.", e); + return null; + } + } + // Getters and Setters public String getId() { @@ -419,10 +446,10 @@ public Instant getLastUpdateTime() { public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public Schedule getSchedule() { + public IntervalSchedule getSchedule() { return this.schedule; } - public void setSchedule(Schedule schedule) { + public void setSchedule(IntervalSchedule schedule) { this.schedule = schedule; } public TIFJobState getState() { @@ -443,10 +470,10 @@ public Instant getLastRefreshedTime() { public void setLastRefreshedTime(Instant lastRefreshedTime) { this.lastRefreshedTime = lastRefreshedTime; } - public String getRefreshType() { + public RefreshType getRefreshType() { return refreshType; } - public void setRefreshType(String refreshType) { + public void setRefreshType(RefreshType refreshType) { this.refreshType = refreshType; } public boolean isEnabled() { @@ -477,6 +504,15 @@ public Map getIocMapStore() { public void setIocMapStore(Map iocMapStore) { this.iocMapStore = iocMapStore; } + + public List getIocTypes() { + return iocTypes; + } + + public void setIocTypes(List iocTypes) { + this.iocTypes = iocTypes; + } + public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { return new SATIFSourceConfigDto(sin); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index a964a1663..2fa5cb199 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -562,7 +562,6 @@ public static TIFJobParameter build(final PutTIFJobRequest request) { ChronoUnit.MINUTES ); return new TIFJobParameter(name, schedule); - } } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java new file mode 100644 index 000000000..6eb669c92 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetTIFSourceConfigAction.java @@ -0,0 +1,51 @@ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestActions; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetTIFSourceConfigAction extends BaseRestHandler { + + private static final Logger log = LogManager.getLogger(RestGetTIFSourceConfigAction.class); + + @Override + public String getName() { + return "get_tif_config_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, String.format(Locale.getDefault(), "%s/{%s}", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID))); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String SaTifSourceConfigId = request.param(SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID, SATIFSourceConfigDto.NO_ID); + + if (SaTifSourceConfigId == null || SaTifSourceConfigId.isEmpty()) { + throw new IllegalArgumentException("missing id"); + } + + SAGetTIFSourceConfigRequest req = new SAGetTIFSourceConfigRequest(SaTifSourceConfigId, RestActions.parseVersion(request)); + + return channel -> client.execute( + SAGetTIFSourceConfigAction.INSTANCE, + req, + new RestToXContentListener<>(channel) + ); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..4e5d15d5c --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestIndexTIFSourceConfigAction.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.RestHandlerUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class RestIndexTIFSourceConfigAction extends BaseRestHandler { + private static final Logger log = LogManager.getLogger(RestIndexTIFSourceConfigAction.class); + @Override + public String getName() { + return "index_tif_config_action"; + } + @Override + public List routes() { + return List.of( + new Route(RestRequest.Method.POST, SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI), + new Route(RestRequest.Method.PUT, String.format(Locale.getDefault(), "%s/{%s}", + SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, SAGetTIFSourceConfigRequest.TIF_SOURCE_CONFIG_ID)) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + log.debug(String.format(Locale.getDefault(), "%s %s", request.method(), SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI)); + + WriteRequest.RefreshPolicy refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE; + if (request.hasParam(RestHandlerUtils.REFRESH)) { + refreshPolicy = WriteRequest.RefreshPolicy.parse(request.param(RestHandlerUtils.REFRESH)); + } + + String id = request.param("feed_id", null); + + XContentParser xcp = request.contentParser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + + SATIFSourceConfigDto tifConfig = SATIFSourceConfigDto.parse(xcp, id, null); + tifConfig.setLastUpdateTime(Instant.now()); + + SAIndexTIFSourceConfigRequest indexTIFConfigRequest = new SAIndexTIFSourceConfigRequest(id, refreshPolicy, request.method(), tifConfig); + return channel -> client.execute(SAIndexTIFSourceConfigAction.INSTANCE, indexTIFConfigRequest, indexTIFConfigResponse(channel, request.method())); + } + + private RestResponseListener indexTIFConfigResponse(RestChannel channel, RestRequest.Method restMethod) { + return new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(SAIndexTIFSourceConfigResponse response) throws Exception { + RestStatus returnStatus = RestStatus.CREATED; + if (restMethod == RestRequest.Method.PUT) { + returnStatus = RestStatus.OK; + } + + BytesRestResponse restResponse = new BytesRestResponse(returnStatus, response.toXContent(channel.newBuilder(), ToXContent.EMPTY_PARAMS)); + + if (restMethod == RestRequest.Method.POST) { + String location = String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, response.getTIFConfigId()); + restResponse.addHeader("Location", location); + } + + return restResponse; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java new file mode 100644 index 000000000..a4f196ea1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigAction.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public class IndexTIFSourceConfigAction { + public static final String INDEX_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/security_analytics/tifSource/write"; + public static final String GET_TIF_SOURCE_CONFIG_ACTION_NAME = "cluster:admin/security_analytics/tifSource/get"; +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java new file mode 100644 index 000000000..db33575eb --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.sacommons; + +/** + * Threat intel feed config creation request interface + */ +public interface IndexTIFSourceConfigRequest { + String getTIFConfigId(); + TIFSourceConfigDto getTIFConfigDto(); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java new file mode 100644 index 000000000..5a9e4daa6 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/IndexTIFSourceConfigResponse.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +public interface IndexTIFSourceConfigResponse { + String getTIFConfigId(); + Long getVersion(); + TIFSourceConfigDto getTIFConfigDto(); +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java index 847fb3be9..822dcd4d4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -1,19 +1,21 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import java.time.Instant; +import java.util.List; import java.util.Map; /** * Threat intel config interface */ public interface TIFSourceConfig { - String getId(); - void setId(String id); + public String getId(); + + public void setId(String id); Long getVersion(); @@ -47,9 +49,9 @@ public interface TIFSourceConfig { void setLastUpdateTime(Instant lastUpdateTime); - Schedule getSchedule(); + IntervalSchedule getSchedule(); - void setSchedule(Schedule schedule); + void setSchedule(IntervalSchedule schedule); TIFJobState getState(); @@ -63,4 +65,8 @@ public interface TIFSourceConfig { void setIocMapStore(Map iocMapStore); + public List getIocTypes(); + + public void setIocTypes(List iocTypes); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java index c8e27d1fa..1899c3af6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java @@ -1,19 +1,21 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; -import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import java.time.Instant; +import java.util.List; import java.util.Map; /** * Threat intel config dto interface */ public interface TIFSourceConfigDto { - String getId(); - void setId(String id); + public String getId(); + + public void setId(String id); Long getVersion(); @@ -47,9 +49,9 @@ public interface TIFSourceConfigDto { void setLastUpdateTime(Instant lastUpdateTime); - Schedule getSchedule(); + IntervalSchedule getSchedule(); - void setSchedule(Schedule schedule); + void setSchedule(IntervalSchedule schedule); TIFJobState getState(); @@ -62,4 +64,8 @@ public interface TIFSourceConfigDto { Map getIocMapStore(); void setIocMapStore(Map iocMapStore); + + public List getIocTypes(); + + public void setIocTypes(List iocTypes); } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java new file mode 100644 index 000000000..9f5438a6e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java @@ -0,0 +1,13 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.core.action.ActionListener; +public abstract class TIFSourceConfigService { + IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener){ + return null; + } + + // TODO: + // update + // delete + // get +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java new file mode 100644 index 000000000..ec0dfb104 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -0,0 +1,121 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.StepListener; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +/** + * Service class for threat intel feed source config object + */ +public class SATIFSourceConfigService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); + private final SATIFSourceConfigDao SaTifSourceConfigDao; + private final TIFLockService lockService; + + /** + * Default constructor + * @param SaTifSourceConfigDao the tif source config dao + * @param lockService the lock service + */ + @Inject + public SATIFSourceConfigService( + final SATIFSourceConfigDao SaTifSourceConfigDao, + final TIFLockService lockService + ) { + this.SaTifSourceConfigDao = SaTifSourceConfigDao; + this.lockService = lockService; + } + + /** + * + * Creates the job index if it doesn't exist and indexes the tif source config object + * + * @param SaTifSourceConfigDto the tif source config dto + * @param lock the lock object + * @param indexTimeout the index time out + * @param listener listener that accepts a tif source config if successful + */ + public void createIndexAndSaveTIFSourceConfig( + final SATIFSourceConfigDto SaTifSourceConfigDto, + final LockModel lock, + final TimeValue indexTimeout, + final ActionListener listener + ) { + try { + SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); + SaTifSourceConfig.setState(TIFJobState.AVAILABLE); + SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, lock, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig response) { + SaTifSourceConfig.setId(response.getId()); + SaTifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void getTIFSourceConfig( + final String SaTifSourceConfigId, + final Long version, + final ActionListener listener + ) { + try { + SaTifSourceConfigDao.getTIFSourceConfig(SaTifSourceConfigId, version, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Converts the DTO to entity + * @param SaTifSourceConfigDto + * @return SaTifSourceConfig + */ + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto SaTifSourceConfigDto) { + return new SATIFSourceConfig( + SaTifSourceConfigDto.getId(), + SaTifSourceConfigDto.getVersion(), + SaTifSourceConfigDto.getName(), + SaTifSourceConfigDto.getFeedFormat(), + SaTifSourceConfigDto.getFeedType(), + SaTifSourceConfigDto.getCreatedByUser(), + SaTifSourceConfigDto.getCreatedAt(), + SaTifSourceConfigDto.getEnabledTime(), + SaTifSourceConfigDto.getLastUpdateTime(), + SaTifSourceConfigDto.getSchedule(), + SaTifSourceConfigDto.getState(), + SaTifSourceConfigDto.getRefreshType(), + SaTifSourceConfigDto.getLastRefreshedTime(), + SaTifSourceConfigDto.getLastRefreshedUser(), + SaTifSourceConfigDto.isEnabled(), + SaTifSourceConfigDto.getIocMapStore(), + SaTifSourceConfigDto.getIocTypes() + ); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java new file mode 100644 index 000000000..93dd34ebc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java @@ -0,0 +1,84 @@ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportGetTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + + private static final Logger log = LogManager.getLogger(TransportGetTIFSourceConfigAction.class); + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + private final SATIFSourceConfigService SaTifConfigService; + + @Inject + public TransportGetTIFSourceConfigAction(TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + final ThreadPool threadPool, + Settings settings, + final SATIFSourceConfigService SaTifConfigService) { + super(SAGetTIFSourceConfigAction.NAME, transportService, actionFilters, SAGetTIFSourceConfigRequest::new); + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + this.SaTifConfigService = SaTifConfigService; + } + + @Override + protected void doExecute(Task task, SAGetTIFSourceConfigRequest request, ActionListener actionListener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + + SaTifConfigService.getTIFSourceConfig(request.getId(), request.getVersion(), new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto(SaTifSourceConfig); + actionListener.onResponse(new SAGetTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto)); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java new file mode 100644 index 000000000..e5a475eea --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.util.ConcurrentModificationException; + +import static org.opensearch.securityanalytics.threatIntel.common.TIFLockService.LOCK_DURATION_IN_SECONDS; +import static org.opensearch.securityanalytics.threatIntel.sacommons.IndexTIFSourceConfigAction.INDEX_TIF_SOURCE_CONFIG_ACTION_NAME; + +/** + * Transport action to create threat intel feeds source config object and save IoCs + */ +public class TransportIndexTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { + private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); + private final SATIFSourceConfigService SaTifSourceConfigService; + private final TIFLockService lockService; + private final ThreadPool threadPool; + private final Settings settings; + private volatile Boolean filterByEnabled; + private final TimeValue indexTimeout; + + + /** + * Default constructor + * @param transportService the transport service + * @param actionFilters the action filters + * @param threadPool the thread pool + * @param lockService the lock service + */ + @Inject + public TransportIndexTIFSourceConfigAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ThreadPool threadPool, + final SATIFSourceConfigService SaTifSourceConfigService, + final TIFLockService lockService, + final Settings settings + ) { + super(SAIndexTIFSourceConfigAction.NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); + this.threadPool = threadPool; + this.SaTifSourceConfigService = SaTifSourceConfigService; + this.lockService = lockService; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); + } + + + @Override + protected void doExecute(final Task task, final SAIndexTIFSourceConfigRequest request, final ActionListener listener) { + // validate user + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + + retrieveLockAndCreateTIFConfig(request, listener, user); + } + + private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest request, ActionListener listener, User user) { + try { + lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { + if (lock == null) { + listener.onFailure( + new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") + ); + log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); + return; + } + try { + SATIFSourceConfigDto SaTifSourceConfigDto = request.getTIFConfigDto(); + if (user != null) { + SaTifSourceConfigDto.setCreatedByUser(user.getName()); + } + try { + SaTifSourceConfigService.createIndexAndSaveTIFSourceConfig(SaTifSourceConfigDto, + lock, + indexTimeout, + new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto(SaTifSourceConfig); + listener.onResponse(new SAIndexTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto)); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + + } catch (Exception e) { + lockService.releaseLock(lock); + listener.onFailure(e); + log.error("listener failed when executing", e); + } + }, exception -> { + listener.onFailure(exception); + log.error("execution failed", exception); + })); + } catch (Exception e) { + log.error("Failed to acquire lock for job", e); + listener.onFailure(e); + } + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index c04c08798..2c756b3d3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -123,7 +123,6 @@ protected void internalDoExecute( listener.onFailure(exception); }); tifJobParameterService.createJobIndexIfNotExists(createIndexStepListener); - } /** diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 59d49f73d..ee258a11d 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,12 +1,14 @@ { - "dynamic": "strict", "_meta" : { "schema_version": 2 }, + "dynamic": "strict", "properties": { - "feed_format_config": { - "dynamic": "false", + "feed_source_config": { "properties": { + "version": { + "type": "long" + }, "feed_name": { "type" : "text", "fields" : { @@ -92,8 +94,16 @@ "enabled": { "type": "boolean" }, - "version": { - "type": "long" + "ioc_map_store": { + "type": "object" + }, + "ioc_types": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } } } }, diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 0b5880bad..91289a91e 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -67,6 +67,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -662,6 +663,9 @@ protected HttpEntity toHttpEntity(UpdateIndexMappingsRequest request) throws IOE protected HttpEntity toHttpEntity(CorrelationRule rule) throws IOException { return new StringEntity(toJsonString(rule), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { + return new StringEntity(toJsonString(SaTifSourceConfigDto), ContentType.APPLICATION_JSON); + } protected RestStatus restStatus(Response response) { return RestStatus.fromCode(response.getStatusLine().getStatusCode()); @@ -706,6 +710,11 @@ protected String toJsonString(ThreatIntelFeedData tifd) throws IOException { return IndexUtilsKt.string(shuffleXContent(tifd.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(SATIFSourceConfigDto SaTifSourceConfigDto) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(SaTifSourceConfigDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String alertingScheduledJobMappings() { return " \"_meta\" : {\n" + " \"schema_version\": 5\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 9fd316ad9..81cff69bf 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -32,6 +32,10 @@ import org.opensearch.securityanalytics.model.IOC; import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.RefreshType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -2826,4 +2830,89 @@ public static IocDto randomIocDto( feedId )); } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto() { + return randomSATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static SATIFSourceConfigDto randomSATIFSourceConfigDto( + String feedName, + String feedFormat, + FeedType feedType, + String createdByUser, + Instant createdAt, + Instant enabledTime, + Instant lastUpdateTime, + org.opensearch.jobscheduler.spi.schedule.IntervalSchedule schedule, + TIFJobState state, + RefreshType refreshType, + Instant lastRefreshedTime, + String lastRefreshedUser, + Boolean isEnabled, + Map iocMapStore, + List iocTypes + ) { + if (feedName == null) { + feedName = randomString(); + } + if (feedFormat == null) { + feedFormat = "STIX"; + } + if (feedType == null) { + feedType = FeedType.INTERNAL; + } + if (isEnabled == null) { + isEnabled = true; + } + if (schedule == null) { + schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + } + if (iocTypes == null) { + iocTypes = List.of("ip"); + } + + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt, + enabledTime, + lastUpdateTime, + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore, + iocTypes + ); + } + + public static XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java new file mode 100644 index 000000000..f0b932472 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class GetTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testGetTIFSourceConfigActionName() { + Assert.assertNotNull(SAGetTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAGetTIFSourceConfigAction.INSTANCE.name(), SAGetTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..376d10b01 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigRequestTests.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.UUIDs; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigRequest; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class GetTIFSourceConfigRequestTests extends OpenSearchTestCase { + public void testStreamInOut() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigRequest newReq = new SAGetTIFSourceConfigRequest(sin); + + assertEquals(id, newReq.getId()); + assertEquals(version, newReq.getVersion()); + } + + public void testValidate() { + String id = UUIDs.base64UUID(); + Long version = 1L; + + SAGetTIFSourceConfigRequest request = new SAGetTIFSourceConfigRequest(id, version); + ActionRequestValidationException e = request.validate(); + assertNull(e); + + request = new SAGetTIFSourceConfigRequest("", 0L); + e = request.validate(); + assertNotNull(e); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..c6e5b08e3 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/GetTIFSourceConfigResponseTests.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class GetTIFSourceConfigResponseTests extends OpenSearchTestCase { + private static final Logger log = LogManager.getLogger(GetTIFSourceConfigResponseTests.class); + + public void testStreamInOut() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + Instant.now(), + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + false, + null, + iocTypes + ); + + SAGetTIFSourceConfigResponse response = new SAGetTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto); + log.error(SaTifSourceConfigDto.getLastUpdateTime()); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAGetTIFSourceConfigResponse newResponse = new SAGetTIFSourceConfigResponse(sin); + + Assert.assertEquals(SaTifSourceConfigDto.getId(), newResponse.getId()); + Assert.assertEquals(SaTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getSaTifSourceConfigDto()); + Assert.assertEquals(feedName, newResponse.getSaTifSourceConfigDto().getName()); + Assert.assertEquals(feedFormat, newResponse.getSaTifSourceConfigDto().getFeedFormat()); + Assert.assertEquals(feedType, newResponse.getSaTifSourceConfigDto().getFeedType()); + Assert.assertEquals(SaTifSourceConfigDto.getState(), newResponse.getSaTifSourceConfigDto().getState()); + Assert.assertEquals(SaTifSourceConfigDto.getEnabledTime(), newResponse.getSaTifSourceConfigDto().getEnabledTime()); + Assert.assertEquals(SaTifSourceConfigDto.getCreatedAt(), newResponse.getSaTifSourceConfigDto().getCreatedAt()); + Assert.assertEquals(SaTifSourceConfigDto.getLastUpdateTime(), newResponse.getSaTifSourceConfigDto().getLastUpdateTime()); + Assert.assertEquals(SaTifSourceConfigDto.isEnabled(), newResponse.getSaTifSourceConfigDto().isEnabled()); + Assert.assertEquals(SaTifSourceConfigDto.getLastRefreshedTime(), newResponse.getSaTifSourceConfigDto().getLastRefreshedTime()); + Assert.assertEquals(SaTifSourceConfigDto.getLastRefreshedUser(), newResponse.getSaTifSourceConfigDto().getLastRefreshedUser()); + Assert.assertEquals(schedule, newResponse.getSaTifSourceConfigDto().getSchedule()); + Assert.assertEquals(SaTifSourceConfigDto.getCreatedByUser(), newResponse.getSaTifSourceConfigDto().getCreatedByUser()); + Assert.assertEquals(SaTifSourceConfigDto.getIocMapStore(), newResponse.getSaTifSourceConfigDto().getIocMapStore()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getSaTifSourceConfigDto().getIocTypes()) && + newResponse.getSaTifSourceConfigDto().getIocTypes().containsAll(iocTypes)); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java new file mode 100644 index 000000000..c8b8b29bd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigActionTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; +import org.opensearch.test.OpenSearchTestCase; + +public class IndexTIFSourceConfigActionTests extends OpenSearchTestCase { + public void testIndexTIFSourceConfigActionName() { + Assert.assertNotNull(SAIndexTIFSourceConfigAction.INSTANCE.name()); + Assert.assertEquals(SAIndexTIFSourceConfigAction.INSTANCE.name(), SAIndexTIFSourceConfigAction.NAME); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java new file mode 100644 index 000000000..21ca175fe --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.rest.RestRequest; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; + +public class IndexTIFSourceConfigRequestTests extends OpenSearchTestCase { + + public void testTIFSourceConfigPostRequest() throws IOException { + SATIFSourceConfigDto SaTifSourceConfigDto = randomSATIFSourceConfigDto(); + String id = SaTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, WriteRequest.RefreshPolicy.IMMEDIATE, RestRequest.Method.POST, SaTifSourceConfigDto); + Assert.assertNotNull(request); + + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigRequest newRequest = new SAIndexTIFSourceConfigRequest(sin); + Assert.assertEquals(id, request.getTIFConfigId()); + Assert.assertEquals(RestRequest.Method.POST, newRequest.getMethod()); + Assert.assertNotNull(newRequest.getTIFConfigDto()); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java new file mode 100644 index 000000000..cce168ae8 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigResponseTests.java @@ -0,0 +1,67 @@ +package org.opensearch.securityanalytics.action; + +import org.junit.Assert; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +public class IndexTIFSourceConfigResponseTests extends OpenSearchTestCase { + + public void testIndexTIFSourceConfigPostResponse() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + SAIndexTIFSourceConfigResponse response = new SAIndexTIFSourceConfigResponse(SaTifSourceConfigDto.getId(), SaTifSourceConfigDto.getVersion(), RestStatus.OK, SaTifSourceConfigDto); + Assert.assertNotNull(response); + + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + SAIndexTIFSourceConfigResponse newResponse = new SAIndexTIFSourceConfigResponse(sin); + + Assert.assertEquals(SaTifSourceConfigDto.getId(), newResponse.getTIFConfigId()); + Assert.assertEquals(SaTifSourceConfigDto.getVersion(), newResponse.getVersion()); + Assert.assertEquals(RestStatus.OK, newResponse.getStatus()); + Assert.assertNotNull(newResponse.getTIFConfigDto()); + Assert.assertEquals(feedName, newResponse.getTIFConfigDto().getName()); + Assert.assertEquals(feedFormat, newResponse.getTIFConfigDto().getFeedFormat()); + Assert.assertEquals(feedType, newResponse.getTIFConfigDto().getFeedType()); + Assert.assertEquals(schedule, newResponse.getTIFConfigDto().getSchedule()); + Assert.assertTrue(iocTypes.containsAll(newResponse.getTIFConfigDto().getIocTypes()) && + newResponse.getTIFConfigDto().getIocTypes().containsAll(iocTypes)); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java new file mode 100644 index 000000000..da9fe8ca2 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.resthandler; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.search.SearchHit; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; + +public class SATIFSourceConfigRestApiIT extends SecurityAnalyticsRestTestCase { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigRestApiIT.class); + public void testCreateSATIFSourceConfig() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(SaTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + } + + public void testGetSATIFSourceConfigById() throws IOException { + String feedName = "test_feed_name"; + String feedFormat = "STIX"; + FeedType feedType = FeedType.INTERNAL; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + List iocTypes = List.of("ip", "dns"); + + SATIFSourceConfigDto SaTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + feedType, + null, + null, + null, + null, + schedule, + null, + null, + null, + null, + true, + null, + iocTypes + ); + + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(SaTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/" + createdId, Collections.emptyMap(), null); + Map getResponse = entityAsMap(response); + + String responseId = responseBody.get("_id").toString(); + Assert.assertEquals("Created Id and returned Id do not match", createdId, responseId); + + int responseVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("Incorrect version", responseVersion > 0); + + String returnedFeedName = (String) ((Map)responseBody.get("tif_config")).get("feed_name"); + Assert.assertEquals("Created feed name and returned feed name do not match", feedName, returnedFeedName); + + String returnedFeedFormat = (String) ((Map)responseBody.get("tif_config")).get("feed_format"); + Assert.assertEquals("Created feed format and returned feed format do not match", feedFormat, returnedFeedFormat); + + String returnedFeedType = (String) ((Map)responseBody.get("tif_config")).get("feed_type"); + Assert.assertEquals("Created feed type and returned feed type do not match", feedType, SATIFSourceConfigDto.toFeedType(returnedFeedType)); + + List returnedIocTypes = (List) ((Map)responseBody.get("tif_config")).get("ioc_types"); + Assert.assertTrue("Created ioc types and returned ioc types do not match", iocTypes.containsAll(returnedIocTypes) && returnedIocTypes.containsAll(iocTypes)); + } +}